4 files changed, 121 insertions, 12 deletions
diff --git a/xlators/cluster/afr/src/afr-common.c b/xlators/cluster/afr/src/afr-common.c
index 59b8038dcac..2fd7879d923 100644
--- a/xlators/cluster/afr/src/afr-common.c
+++ b/xlators/cluster/afr/src/afr-common.c
@@ -1188,6 +1188,62 @@ afr_handle_quota_size (afr_local_t *local, xlator_t *this,
         }
 }
 
+static int
+afr_lookup_select_parent_read_child (xlator_t *this, inode_t *parent,
+                                     afr_local_t *local)
+{
+        int             i                   = 0;
+        int             child_index         = -1;
+        int             par_read_child      = -1;
+        int             par_read_child_iter = -1;
+        int            *fresh_children      = NULL;
+        int            *success_children    = NULL;
+        afr_private_t  *priv                = NULL;
+
+        priv = this->private;
+        success_children = local->cont.lookup.success_children;
+
+        if (!parent)
+                return 0;
+
+        fresh_children = afr_children_create (priv->child_count);
+
+        par_read_child = afr_inode_get_read_ctx (this, parent, fresh_children);
+
+        for (i = 0; i < priv->child_count; i++) {
+                child_index = success_children[i];
+
+                if (child_index == -1)
+                        break;
+
+                if (par_read_child_iter == -1) {
+                        par_read_child_iter = child_index;
+                        continue;
+                }
+
+                if ((par_read_child_iter != par_read_child) && fresh_children &&
+                    (afr_is_child_present (fresh_children, priv->child_count,
+                                           child_index)))
+                        par_read_child_iter = child_index;
+
+                if (child_index == par_read_child)
+                        par_read_child_iter = child_index;
+        }
+
+        /* At the end of the for-loop, the only reason why @par_read_child_iter
+         * could be -1 is when this LOOKUP has failed on all sub-volumes.
+         * So it is okay to send an arbitrary subvolume (0 in this case)
+         * as parent read child.
+         */
+
+        if (par_read_child_iter == -1)
+                par_read_child_iter = 0;
+
+        GF_FREE (fresh_children);
+        return par_read_child_iter;
+
+}
+
 int
 afr_lookup_build_response_params (afr_local_t *local, xlator_t *this)
 {
@@ -1198,8 +1254,10 @@ afr_lookup_build_response_params (afr_local_t *local, xlator_t *this)
         int32_t         *sources = NULL;
         afr_private_t   *priv = NULL;
         int32_t         read_child = -1;
+        int32_t         par_read_child = -1;
         int             ret = 0;
         int             i = 0;
+        inode_t        *parent = NULL;
 
         GF_ASSERT (local);
 
@@ -1207,6 +1265,7 @@ afr_lookup_build_response_params (afr_local_t *local, xlator_t *this)
         postparent = &local->cont.lookup.postparent;
         xattr = &local->cont.lookup.xattr;
         priv = this->private;
+        parent = local->loc.parent;
 
         read_child = afr_inode_get_read_ctx (this, local->cont.lookup.inode,
                                              local->fresh_children);
@@ -1239,7 +1298,11 @@ afr_lookup_build_response_params (afr_local_t *local, xlator_t *this)
                 *xattr = dict_ref (local->cont.lookup.xattrs[read_child]);
 
         *buf = local->cont.lookup.bufs[read_child];
-        *postparent = local->cont.lookup.postparents[read_child];
+
+        par_read_child = afr_lookup_select_parent_read_child (this, parent,
+                                                              local);
+
+        *postparent = local->cont.lookup.postparents[par_read_child];
 
         if (dict_get (local->xattr_req, QUOTA_SIZE_KEY))
                 afr_handle_quota_size (local, this, *xattr);
diff --git a/xlators/cluster/afr/src/afr-dir-read.c b/xlators/cluster/afr/src/afr-dir-read.c
index 689dd84e646..d94cb0ca699 100644
--- a/xlators/cluster/afr/src/afr-dir-read.c
+++ b/xlators/cluster/afr/src/afr-dir-read.c
@@ -422,14 +422,32 @@ afr_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
                   dict_t *xdata)
 {
-        afr_local_t     *local = NULL;
+        afr_local_t     *local          = NULL;
+        gf_dirent_t     *entry          = NULL;
+        int              par_read_child = (long) cookie;
+        int32_t          read_child     = -1;
 
         if (op_ret == -1)
                 goto out;
 
         local = frame->local;
+
         afr_readdir_filter_trash_dir (entries, local->fd);
 
+        list_for_each_entry (entry, &entries->list, list) {
+                if (entry->inode) {
+                        read_child = -1;
+
+                        read_child = afr_inode_get_read_ctx (this, entry->inode,
+                                                             NULL);
+                        if (read_child != par_read_child) {
+                                inode_unref (entry->inode);
+                                entry->inode = NULL;
+                                continue;
+                        }
+                }
+        }
+
 out:
         AFR_STACK_UNWIND (readdirp, frame, op_ret, op_errno, entries, NULL);
         return 0;
diff --git a/xlators/cluster/afr/src/afr-dir-write.c b/xlators/cluster/afr/src/afr-dir-write.c
index 3cdec64536d..f996181cd2e 100644
--- a/xlators/cluster/afr/src/afr-dir-write.c
+++ b/xlators/cluster/afr/src/afr-dir-write.c
@@ -80,26 +80,35 @@ __dir_entry_fop_common_cbk (call_frame_t *frame, int child_index,
                             struct iatt *postparent, struct iatt *prenewparent,
                             struct iatt *postnewparent)
 {
-        afr_local_t     *local          = NULL;
+        afr_local_t     *local              = NULL;
+        inode_t         *parent             = NULL;
+        inode_t         *parent2            = NULL;
 
         local = frame->local;
 
+        parent = local->loc.parent;
+        parent2 = local->newloc.parent;
+
         if (afr_fop_failed (op_ret, op_errno))
                 afr_transaction_fop_failed (frame, this, child_index);
 
         if (op_ret > -1) {
                 local->op_ret = op_ret;
 
+                AFR_UPDATE_PARENT_BUF (parent, this, child_index, local,
+                                       local->cont.dir_fop.preparent,
+                                       local->cont.dir_fop.postparent,
+                                       preparent, postparent);
+
+                AFR_UPDATE_PARENT_BUF (parent2, this, child_index, local,
+                                       local->cont.dir_fop.prenewparent,
+                                       local->cont.dir_fop.postnewparent,
+                                       prenewparent, postnewparent);
+
                 if ((local->success_count == 0) ||
                     (child_index == local->read_child_index)) {
-                        local->cont.dir_fop.preparent      = *preparent;
-                        local->cont.dir_fop.postparent     = *postparent;
                         if (buf)
                                 local->cont.dir_fop.buf            = *buf;
-                        if (prenewparent)
-                             local->cont.dir_fop.prenewparent  = *prenewparent;
-                        if (postnewparent)
-                             local->cont.dir_fop.postnewparent = *postnewparent;
                 }
 
                 local->cont.dir_fop.inode = inode;
@@ -934,8 +943,8 @@ afr_link_unwind (call_frame_t *frame, xlator_t *this)
                                   local->op_ret, local->op_errno,
                                   local->cont.dir_fop.inode,
                                   &local->cont.dir_fop.buf,
-                                  &local->cont.dir_fop.preparent,
-                                  &local->cont.dir_fop.postparent,
+                                  &local->cont.dir_fop.prenewparent,
+                                  &local->cont.dir_fop.postnewparent,
                                   NULL);
         }
 
@@ -958,7 +967,7 @@ afr_link_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         {
                 __dir_entry_fop_common_cbk (frame, child_index, this,
                                             op_ret, op_errno, inode, buf,
-                                            preparent, postparent, NULL, NULL);
+                                            NULL, NULL, preparent, postparent);
         }
         UNLOCK (&frame->lock);
 
diff --git a/xlators/cluster/afr/src/afr.h b/xlators/cluster/afr/src/afr.h
index 87ad67c6384..2548900127f 100644
--- a/xlators/cluster/afr/src/afr.h
+++ b/xlators/cluster/afr/src/afr.h
@@ -1195,6 +1195,25 @@ afr_xattr_array_destroy (dict_t **xattr, unsigned int child_count);
         }                                                               \
 } while (0)
 
+#define AFR_UPDATE_PARENT_BUF(parent, this, child_index, local,             \
+                               dst_preparent, dst_postparent,               \
+                               src_preparent, src_postparent) do {          \
+        int __par_read_child = -1;                                          \
+        if (parent) {                                                       \
+                __par_read_child = afr_inode_get_read_ctx (this, parent,    \
+                                                           NULL);           \
+                if (__par_read_child < 0)                                   \
+                        __par_read_child = local->read_child_index;         \
+                if ((local->success_count == 0) ||                          \
+                    (__par_read_child == child_index)) {                    \
+                        if (src_preparent)                                  \
+                                dst_preparent = *src_preparent;             \
+                        if (src_postparent)                                 \
+                                dst_postparent = *src_postparent;           \
+                }                                                           \
+        }                                                                   \
+} while (0)
+
 int
 afr_fd_report_unstable_write (xlator_t *this, fd_t *fd);
 
diff --git a/xlators/Makefile.am b/xlators/Makefile.am
index b1643d26c..f60fa85ce 100644
--- a/xlators/Makefile.am
+++ b/xlators/Makefile.am
@@ -1,3 +1,4 @@
-SUBDIRS = cluster storage protocol performance debug features encryption mount nfs mgmt system
+SUBDIRS = cluster storage protocol performance debug features encryption mount nfs mgmt system \
+          playground
 
 CLEANFILES =
diff --git a/xlators/bindings/Makefile.am b/xlators/bindings/Makefile.am
deleted file mode 100644
index f77665802..000000000
--- a/xlators/bindings/Makefile.am
+++ /dev/null
@@ -1 +0,0 @@
-SUBDIRS = $(BINDINGS_SUBDIRS)
diff --git a/xlators/bindings/python/src/Makefile.am b/xlators/bindings/python/src/Makefile.am
deleted file mode 100644
index c0b9141c6..000000000
--- a/xlators/bindings/python/src/Makefile.am
+++ /dev/null
@@ -1,19 +0,0 @@
-
-xlator_PROGRAMS = python.so
-
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/bindings
-
-python_PYTHON = gluster.py glustertypes.py glusterstack.py
-
-pythondir = $(xlatordir)/python
-
-python_so_SOURCES = python.c
-
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall \
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles \
-	$(PYTHON_CPPLAGS) -DGLUSTER_PYTHON_PATH=\"$(pythondir)\"
-
-AM_LDFLAGS = $(PYTHON_LDFLAGS)
-
-CLEANFILES = 
-
diff --git a/xlators/bindings/python/src/gluster.py b/xlators/bindings/python/src/gluster.py
deleted file mode 100644
index ee0eb1310..000000000
--- a/xlators/bindings/python/src/gluster.py
+++ /dev/null
@@ -1,47 +0,0 @@
-#   Copyright (c) 2007 Chris AtLee <chris@atlee.ca>
-#   This file is part of GlusterFS.
-#
-#   GlusterFS is free software; you can redistribute it and/or modify
-#   it under the terms of the GNU General Public License as published
-#   by the Free Software Foundation; either version 3 of the License,
-#   or (at your option) any later version.
-#
-#   GlusterFS is distributed in the hope that it will be useful, but
-#   WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-#   General Public License for more details.
-#
-#   You should have received a copy of the GNU General Public License
-#   along with this program.  If not, see
-#   <http://www.gnu.org/licenses/>.
-from ctypes import *
-from glustertypes import *
-from glusterstack import *
-import sys
-import inspect
-
-libglusterfs = CDLL("libglusterfs.so")
-_gf_log = libglusterfs._gf_log
-_gf_log.restype = c_int32
-_gf_log.argtypes = [c_char_p, c_char_p, c_char_p, c_int32, c_int, c_char_p]
-
-gf_log_loglevel = c_int.in_dll(libglusterfs, "gf_log_loglevel")
-
-GF_LOG_NONE = 0
-GF_LOG_CRITICAL = 1
-GF_LOG_ERROR = 2
-GF_LOG_WARNING = 3
-GF_LOG_DEBUG = 4
-
-def gf_log(module, level, fmt, *params):
-    if level <= gf_log_loglevel:
-        frame = sys._getframe(1)
-        _gf_log(module, frame.f_code.co_filename, frame.f_code.co_name,
-                frame.f_lineno, level, fmt, *params)
-
-class ComplexTranslator(object):
-    def __init__(self, xlator):
-        self.xlator = xlator_t.from_address(xlator)
-
-    def __getattr__(self, item):
-        return getattr(self.xlator, item)
diff --git a/xlators/bindings/python/src/glusterstack.py b/xlators/bindings/python/src/glusterstack.py
deleted file mode 100644
index ba24c8165..000000000
--- a/xlators/bindings/python/src/glusterstack.py
+++ /dev/null
@@ -1,55 +0,0 @@
-#   Copyright (c) 2007 Chris AtLee <chris@atlee.ca>
-#   This file is part of GlusterFS.
-#
-#   GlusterFS is free software; you can redistribute it and/or modify
-#   it under the terms of the GNU General Public License as published
-#   by the Free Software Foundation; either version 3 of the License,
-#   or (at your option) any later version.
-#
-#   GlusterFS is distributed in the hope that it will be useful, but
-#   WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-#   General Public License for more details.
-#
-#   You should have received a copy of the GNU General Public License
-#   along with this program.  If not, see
-#   <http://www.gnu.org/licenses/>.
-from ctypes import *
-from glustertypes import *
-
-libc = CDLL("libc.so.6")
-calloc = libc.calloc
-calloc.argtypes = [c_int, c_int]
-calloc.restype = c_void_p
-
-# TODO: Can these be done in C somehow?
-def stack_wind(frame, rfn, obj, fn, *params):
-    """Frame is a frame object"""
-    _new = cast(calloc(1, sizeof(call_frame_t)), POINTER(call_frame_t))
-    _new[0].root = frame.root
-    _new[0].next = frame.root[0].frames.next
-    _new[0].prev = pointer(frame.root[0].frames)
-    if frame.root[0].frames.next:
-        frame.root[0].frames.next[0].prev = _new
-    frame.root[0].frames.next = _new
-    _new[0].this = obj
-    # TODO: Type checking like tmp_cbk?
-    _new[0].ret = rfn
-    _new[0].parent = pointer(frame)
-    _new[0].cookie = cast(_new, c_void_p)
-    # TODO: Initialize lock
-    #_new.lock.init()
-    frame.ref_count += 1
-    fn(_new, obj, *params)
-
-def stack_unwind(frame, *params):
-    """Frame is a frame object"""
-    fn = frame[0].ret
-    parent = frame[0].parent[0]
-    parent.ref_count -= 1
-
-    op_ret = params[0]
-    op_err = params[1]
-    params = params[2:]
-    fn(parent, call_frame_t.from_address(frame[0].cookie), parent.this,
-            op_ret, op_err, *params)
diff --git a/xlators/bindings/python/src/glustertypes.py b/xlators/bindings/python/src/glustertypes.py
deleted file mode 100644
index e9069d07c..000000000
--- a/xlators/bindings/python/src/glustertypes.py
+++ /dev/null
@@ -1,167 +0,0 @@
-#   Copyright (c) 2007 Chris AtLee <chris@atlee.ca>
-#   This file is part of GlusterFS.
-#
-#   GlusterFS is free software; you can redistribute it and/or modify
-#   it under the terms of the GNU General Public License as published
-#   by the Free Software Foundation; either version 3 of the License,
-#   or (at your option) any later version.
-#
-#   GlusterFS is distributed in the hope that it will be useful, but
-#   WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-#   General Public License for more details.
-#
-#   You should have received a copy of the GNU General Public License
-#   along with this program.  If not, see
-#   <http://www.gnu.org/licenses/>.
-from ctypes import *
-import collections
-
-#
-# Forward declaration of some gluster types
-#
-class call_frame_t(Structure):
-    pass
-
-class call_ctx_t(Structure):
-    pass
-
-class call_pool_t(Structure):
-    pass
-
-class xlator_t(Structure):
-    def _getFirstChild(self):
-        return self.children[0].xlator
-    firstChild = property(_getFirstChild)
-
-class xlator_list_t(Structure):
-    pass
-
-class xlator_fops(Structure):
-    pass
-
-class xlator_mops(Structure):
-    pass
-
-class glusterfs_ctx_t(Structure):
-    pass
-
-class list_head(Structure):
-    pass
-
-class dict_t(Structure):
-    pass
-
-class inode_table_t(Structure):
-    pass
-
-class fd_t(Structure):
-    pass
-
-class iovec(Structure):
-    _fields_ = [
-            ("iov_base", c_void_p),
-            ("iov_len", c_size_t),
-            ]
-
-    def __init__(self, s):
-        self.iov_base = cast(c_char_p(s), c_void_p)
-        self.iov_len = len(s)
-
-    def getBytes(self):
-        return string_at(self.iov_base, self.iov_len)
-
-# This is a pthread_spinlock_t
-# TODO: what happens to volatile-ness?
-gf_lock_t = c_int
-
-uid_t = c_uint32
-gid_t = c_uint32
-pid_t = c_int32
-
-off_t = c_int64
-
-#
-# Function pointer types
-#
-ret_fn_t = CFUNCTYPE(c_int32, POINTER(call_frame_t), POINTER(call_frame_t),
-                              POINTER(xlator_t), c_int32, c_int32)
-
-fini_fn_t = CFUNCTYPE(None, POINTER(xlator_t))
-init_fn_t = CFUNCTYPE(c_int32, POINTER(xlator_t))
-event_notify_fn_t = CFUNCTYPE(c_int32, POINTER(xlator_t), c_int32, c_void_p)
-
-list_head._fields_ = [
-        ("next", POINTER(list_head)),
-        ("prev", POINTER(list_head)),
-        ]
-
-call_frame_t._fields_ = [
-        ("root", POINTER(call_ctx_t)),
-        ("parent", POINTER(call_frame_t)),
-        ("next", POINTER(call_frame_t)),
-        ("prev", POINTER(call_frame_t)),
-        ("local", c_void_p),
-        ("this", POINTER(xlator_t)),
-        ("ret", ret_fn_t),
-        ("ref_count", c_int32),
-        ("lock", gf_lock_t),
-        ("cookie", c_void_p),
-        ("op", c_int32),
-        ("type", c_int8),
-        ]
-
-call_ctx_t._fields_ = [
-        ("all_frames", list_head),
-        ("trans", c_void_p),
-        ("pool", call_pool_t),
-        ("unique", c_uint64),
-        ("state", c_void_p),
-        ("uid", uid_t),
-        ("gid", gid_t),
-        ("pid", pid_t),
-        ("frames", call_frame_t),
-        ("req_refs", POINTER(dict_t)),
-        ("rsp_refs", POINTER(dict_t)),
-        ]
-
-xlator_t._fields_ = [
-        ("name", c_char_p),
-        ("type", c_char_p),
-        ("next", POINTER(xlator_t)),
-        ("prev", POINTER(xlator_t)),
-        ("parent", POINTER(xlator_t)),
-        ("children", POINTER(xlator_list_t)),
-        ("fops", POINTER(xlator_fops)),
-        ("mops", POINTER(xlator_mops)),
-        ("fini", fini_fn_t),
-        ("init", init_fn_t),
-        ("notify", event_notify_fn_t),
-        ("options", POINTER(dict_t)),
-        ("ctx", POINTER(glusterfs_ctx_t)),
-        ("itable", POINTER(inode_table_t)),
-        ("ready", c_char),
-        ("private", c_void_p),
-        ]
-
-xlator_list_t._fields_ = [
-        ("xlator", POINTER(xlator_t)),
-        ("next", POINTER(xlator_list_t)),
-        ]
-
-fop_functions = collections.defaultdict(lambda: c_void_p)
-fop_function_names = ['lookup', 'forget', 'stat', 'fstat', 'chmod', 'fchmod',
-        'chown', 'fchown', 'truncate', 'ftruncate', 'utimens', 'access',
-        'readlink', 'mknod', 'mkdir', 'unlink', 'rmdir', 'symlink',
-        'rename', 'link', 'create', 'open', 'readv', 'writev', 'flush',
-        'close', 'fsync', 'opendir', 'readdir', 'closedir', 'fsyncdir',
-        'statfs', 'setxattr', 'getxattr', 'removexattr', 'lk', 'writedir',
-        # TODO: Call backs?
-        ]
-
-fop_writev_t = CFUNCTYPE(c_int32, POINTER(call_frame_t), POINTER(xlator_t),
-                                  POINTER(fd_t), POINTER(iovec), c_int32,
-                                  off_t)
-
-fop_functions['writev'] = fop_writev_t
-xlator_fops._fields_ = [(f, fop_functions[f]) for f in fop_function_names]
diff --git a/xlators/bindings/python/src/python.c b/xlators/bindings/python/src/python.c
deleted file mode 100644
index 3310a2115..000000000
--- a/xlators/bindings/python/src/python.c
+++ /dev/null
@@ -1,232 +0,0 @@
-/*
-   Copyright (c) 2007-2010 Chris AtLee <chris@atlee.ca>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-#include <Python.h>
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "xlator.h"
-#include "logging.h"
-#include "defaults.h"
-
-typedef struct
-{
-    char        *scriptname;
-    PyObject    *pXlator;
-    PyObject    *pScriptModule;
-    PyObject    *pGlusterModule;
-    PyThreadState *pInterp;
-
-    PyObject    *pFrameType, *pVectorType, *pFdType;
-} python_private_t;
-
-int32_t
-python_writev (call_frame_t *frame,
-              xlator_t *this,
-              fd_t *fd,
-              struct iovec *vector,
-              int32_t count, 
-              off_t offset)
-{
-  python_private_t *priv = (python_private_t *)this->private;
-  gf_log("python", GF_LOG_DEBUG, "In writev");
-  if (PyObject_HasAttrString(priv->pXlator, "writev"))
-  {
-
-      PyObject *retval = PyObject_CallMethod(priv->pXlator, "writev",
-              "O O O i l",
-              PyObject_CallMethod(priv->pFrameType, "from_address", "O&", PyLong_FromVoidPtr, frame),
-              PyObject_CallMethod(priv->pFdType, "from_address", "O&", PyLong_FromVoidPtr, fd),
-              PyObject_CallMethod(priv->pVectorType, "from_address", "O&", PyLong_FromVoidPtr, vector),
-              count,
-              offset);
-      if (PyErr_Occurred())
-      {
-          PyErr_Print();
-      }
-      Py_XDECREF(retval);
-  }
-  else
-  {
-      return default_writev(frame, this, fd, vector, count, offset);
-  }
-  return 0;
-}
-
-struct xlator_fops fops = {
-    .writev       = python_writev
-};
-
-static PyObject *
-AnonModule_FromFile (const char* fname)
-{
-    // Get the builtins
-    PyThreadState* pThread = PyThreadState_Get();
-    PyObject *pBuiltins = pThread->interp->builtins;
-
-    if (PyErr_Occurred())
-    {
-        PyErr_Print();
-        return NULL;
-    }
-
-    // Create a new dictionary for running code in
-    PyObject *pModuleDict = PyDict_New();
-    PyDict_SetItemString(pModuleDict, "__builtins__", pBuiltins);
-    Py_INCREF(pBuiltins);
-
-    // Run the file in the new context
-    FILE* fp = fopen(fname, "r");
-    PyRun_File(fp, fname, Py_file_input, pModuleDict, pModuleDict);
-    fclose(fp);
-    if (PyErr_Occurred())
-    {
-        PyErr_Print();
-        Py_DECREF(pModuleDict);
-        Py_DECREF(pBuiltins);
-        return NULL;
-    }
-
-    // Create an object to hold the new context
-    PyRun_String("class ModuleWrapper(object):\n\tpass\n", Py_single_input, pModuleDict, pModuleDict);
-    if (PyErr_Occurred())
-    {
-        PyErr_Print();
-        Py_DECREF(pModuleDict);
-        Py_DECREF(pBuiltins);
-        return NULL;
-    }
-    PyObject *pModule = PyRun_String("ModuleWrapper()", Py_eval_input, pModuleDict, pModuleDict);
-    if (PyErr_Occurred())
-    {
-        PyErr_Print();
-        Py_DECREF(pModuleDict);
-        Py_DECREF(pBuiltins);
-        Py_XDECREF(pModule);
-        return NULL;
-    }
-
-    // Set the new context's dictionary to the one we used to run the code
-    // inside
-    PyObject_SetAttrString(pModule, "__dict__", pModuleDict);
-    if (PyErr_Occurred())
-    {
-        PyErr_Print();
-        Py_DECREF(pModuleDict);
-        Py_DECREF(pBuiltins);
-        Py_DECREF(pModule);
-        return NULL;
-    }
-
-    return pModule;
-}
-
-int32_t
-init (xlator_t *this)
-{
-  // This is ok to call more than once per process
-  Py_InitializeEx(0);
-
-  if (!this->children) {
-    gf_log ("python", GF_LOG_ERROR, 
-            "FATAL: python should have exactly one child");
-    return -1;
-  }
-
-  python_private_t *priv = CALLOC (sizeof (python_private_t), 1);
-  ERR_ABORT (priv);
-
-  data_t *scriptname = dict_get (this->options, "scriptname");
-  if (scriptname) {
-      priv->scriptname = data_to_str(scriptname);
-  } else {
-      gf_log("python", GF_LOG_ERROR,
-              "FATAL: python requires the scriptname parameter");
-      return -1;
-  }
-
-  priv->pInterp = Py_NewInterpreter();
-    
-  // Adjust python's path
-  PyObject *syspath = PySys_GetObject("path");
-  PyObject *path = PyString_FromString(GLUSTER_PYTHON_PATH);
-  PyList_Append(syspath, path);
-  Py_DECREF(path);
-
-  gf_log("python", GF_LOG_DEBUG,
-          "Loading gluster module");
-
-  priv->pGlusterModule = PyImport_ImportModule("gluster");
-  if (PyErr_Occurred())
-  {
-      PyErr_Print();
-      return -1;
-  }
-
-  priv->pFrameType = PyObject_GetAttrString(priv->pGlusterModule, "call_frame_t");
-  priv->pFdType = PyObject_GetAttrString(priv->pGlusterModule, "fd_t");
-  priv->pVectorType = PyObject_GetAttrString(priv->pGlusterModule, "iovec");
-
-  gf_log("python", GF_LOG_DEBUG, "Loading script...%s", priv->scriptname);
-  
-  priv->pScriptModule = AnonModule_FromFile(priv->scriptname);
-  if (!priv->pScriptModule || PyErr_Occurred())
-  {
-      gf_log("python", GF_LOG_ERROR, "Error loading %s", priv->scriptname);
-      PyErr_Print();
-      return -1;
-  }
-
-  if (!PyObject_HasAttrString(priv->pScriptModule, "xlator"))
-  {
-      gf_log("python", GF_LOG_ERROR, "%s does not have a xlator attribute", priv->scriptname);
-      return -1;
-  }
-  gf_log("python", GF_LOG_DEBUG, "Instantiating translator");
-  priv->pXlator = PyObject_CallMethod(priv->pScriptModule, "xlator", "O&",
-          PyLong_FromVoidPtr, this);
-  if (PyErr_Occurred() || !priv->pXlator)
-  {
-      PyErr_Print();
-      return -1;
-  }
-
-  this->private = priv;
-
-  gf_log ("python", GF_LOG_DEBUG, "python xlator loaded");
-  return 0;
-}
-
-void 
-fini (xlator_t *this)
-{
-  python_private_t *priv = (python_private_t*)(this->private);
-  Py_DECREF(priv->pXlator);
-  Py_DECREF(priv->pScriptModule);
-  Py_DECREF(priv->pGlusterModule);
-  Py_DECREF(priv->pFrameType);
-  Py_DECREF(priv->pFdType);
-  Py_DECREF(priv->pVectorType);
-  Py_EndInterpreter(priv->pInterp);
-  return;
-}
diff --git a/xlators/bindings/python/src/testxlator.py b/xlators/bindings/python/src/testxlator.py
deleted file mode 100644
index 507455c85..000000000
--- a/xlators/bindings/python/src/testxlator.py
+++ /dev/null
@@ -1,56 +0,0 @@
-#   Copyright (c) 2007 Chris AtLee <chris@atlee.ca>
-#   This file is part of GlusterFS.
-#
-#   GlusterFS is free software; you can redistribute it and/or modify
-#   it under the terms of the GNU General Public License as published
-#   by the Free Software Foundation; either version 3 of the License,
-#   or (at your option) any later version.
-#
-#   GlusterFS is distributed in the hope that it will be useful, but
-#   WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-#   General Public License for more details.
-#
-#   You should have received a copy of the GNU General Public License
-#   along with this program.  If not, see
-#   <http://www.gnu.org/licenses/>.
-
-"""
-This is a test translator written in python.
-
-Important things to note:
-    This file must be import-able from glusterfsd.  This probably means
-    setting PYTHONPATH to where this file is located.
-
-    This file must have a top-level xlator class object that will be
-    used to instantiate individual translators.
-"""
-from gluster import *
-
-class MyXlator(ComplexTranslator):
-    name = "MyXlator"
-    def writev_cbk(self, frame, cookie, op_ret, op_errno, buf):
-        stack_unwind(frame, op_ret, op_errno, buf)
-        return 0
-
-    def writev(self, frame, fd, vector, count, offset):
-        gf_log(self.name, GF_LOG_WARNING, "writev %i bytes", vector.iov_len)
-        # TODO: Use cookie to pass this to writev_cbk
-        old_count = vector.iov_len
-
-        data = vector.getBytes().encode("zlib")
-
-        vector = iovec(data)
-        gf_log(self.name, GF_LOG_WARNING, "writev %i bytes", vector.iov_len)
-
-        @ret_fn_t
-        def rfn(frame, prev, this, op_ret, op_errno, *params):
-            if len(params) == 0:
-                params = [0]
-            return self.writev_cbk(frame, prev, old_count, op_errno, *params)
-
-        stack_wind(frame, rfn, self.firstChild,
-                self.firstChild[0].fops[0].writev, fd, vector, count, offset)
-        return 0
-
-xlator = MyXlator
diff --git a/xlators/cluster/afr/src/Makefile.am b/xlators/cluster/afr/src/Makefile.am
index 95db5dd96..ea5a90abb 100644
--- a/xlators/cluster/afr/src/Makefile.am
+++ b/xlators/cluster/afr/src/Makefile.am
@@ -2,34 +2,37 @@ xlator_LTLIBRARIES = afr.la pump.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/cluster
 
 afr_common_source = afr-dir-read.c afr-dir-write.c afr-inode-read.c \
-	afr-inode-write.c afr-open.c afr-transaction.c afr-self-heal-data.c \
-	afr-self-heal-common.c afr-self-heal-metadata.c afr-self-heal-entry.c \
-	afr-self-heal-algorithm.c afr-lk-common.c afr-self-heald.c \
+	afr-inode-write.c afr-open.c afr-transaction.c afr-lk-common.c \
+	afr-read-txn.c \
 	$(top_builddir)/xlators/lib/src/libxlator.c
 
-afr_la_LDFLAGS = -module -avoidversion
-afr_la_SOURCES = $(afr_common_source) afr.c
+AFR_SELFHEAL_SOURCES = afr-self-heal-common.c afr-self-heal-data.c \
+	afr-self-heal-entry.c afr-self-heal-metadata.c afr-self-heald.c \
+	afr-self-heal-name.c
+
+afr_la_LDFLAGS = -module -avoid-version
+afr_la_SOURCES = $(afr_common_source) $(AFR_SELFHEAL_SOURCES) afr.c
 afr_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-pump_la_LDFLAGS = -module -avoidversion
-pump_la_SOURCES =  $(afr_common_source) pump.c
+pump_la_LDFLAGS = -module -avoid-version
+pump_la_SOURCES =  $(afr_common_source) $(AFR_SELFHEAL_SOURCES) pump.c
 pump_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = afr.h afr-transaction.h afr-inode-write.h afr-inode-read.h \
-	afr-dir-read.h afr-dir-write.h afr-self-heal.h afr-self-heal-common.h \
-	afr-self-heal-algorithm.h pump.h afr-mem-types.h afr-common.c \
-	afr-self-heald.h $(top_builddir)/xlators/lib/src/libxlator.h \
-	$(top_builddir)/glusterfsd/src/glusterfsd.h
+	afr-dir-read.h afr-dir-write.h afr-self-heal.h afr-mem-types.h \
+	afr-common.c afr-self-heald.h pump.h \
+	$(top_builddir)/xlators/lib/src/libxlator.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
+AM_CPPFLAGS = $(GF_CPPFLAGS) \
 	-I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/xlators/lib/src \
-	-I$(top_srcdir)/rpc/rpc-lib/src -shared -nostartfiles $(GF_CFLAGS)
+	-I$(top_srcdir)/rpc/rpc-lib/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES =
 
 uninstall-local:
 	rm -f $(DESTDIR)$(xlatordir)/replicate.so
-	rm -f $(DESTDIR)$(xlatordir)/pump.so
 
 install-data-hook:
 	ln -sf afr.so $(DESTDIR)$(xlatordir)/replicate.so
diff --git a/xlators/cluster/afr/src/afr-common.c b/xlators/cluster/afr/src/afr-common.c
index cefbaa8ec..6bd231600 100644
--- a/xlators/cluster/afr/src/afr-common.c
+++ b/xlators/cluster/afr/src/afr-common.c
@@ -45,775 +45,821 @@
 #include "afr-dir-write.h"
 #include "afr-transaction.h"
 #include "afr-self-heal.h"
-#include "afr-self-heal-common.h"
 #include "afr-self-heald.h"
-#include "pump.h"
 
-#define AFR_ICTX_OPENDIR_DONE_MASK     0x0000000200000000ULL
-#define AFR_ICTX_SPLIT_BRAIN_MASK      0x0000000100000000ULL
-#define AFR_ICTX_READ_CHILD_MASK       0x00000000FFFFFFFFULL
 
-int
-afr_lookup_done_success_action (call_frame_t *frame, xlator_t *this,
-                                gf_boolean_t fail_conflict);
-void
-afr_children_copy (int32_t *dst, int32_t *src, unsigned int child_count)
+call_frame_t *
+afr_copy_frame (call_frame_t *base)
 {
-        int     i = 0;
+	afr_local_t *local = NULL;
+	call_frame_t *frame = NULL;
+	int op_errno = 0;
 
-        for (i = 0; i < child_count; i++)
-                dst[i] = src[i];
+	frame = copy_frame (base);
+	if (!frame)
+		return NULL;
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local) {
+		AFR_STACK_DESTROY (frame);
+		return NULL;
+	}
+
+	return frame;
 }
 
-void
-afr_xattr_req_prepare (xlator_t *this, dict_t *xattr_req, const char *path)
-{
-        int             i           = 0;
-        afr_private_t   *priv       = NULL;
-        int             ret         = 0;
+/*
+ * INODE CTX 64-bit VALUE FORMAT FOR SMALL (<= 16) SUBVOL COUNTS:
+ *
+ * |<----------   64bit   ------------>|
+ *  63           32 31    16 15       0
+ * |   EVENT_GEN   |  DATA  | METADATA |
+ *
+ *
+ *  METADATA (bit-0 .. bit-15): bitmap representing subvolumes from which
+ *                              metadata can be attempted to be read.
+ *
+ *                              bit-0 => priv->subvolumes[0]
+ *                              bit-1 => priv->subvolumes[1]
+ *                              ... etc. till bit-15
+ *
+ *  DATA (bit-16 .. bit-31): bitmap representing subvolumes from which data
+ *                           can be attempted to be read.
+ *
+ *                           bit-16 => priv->subvolumes[0]
+ *                           bit-17 => priv->subvolumes[1]
+ *                           ... etc. till bit-31
+ *
+ *  EVENT_GEN (bit-32 .. bit-63): event generation (i.e priv->event_generation)
+ *                                when DATA and METADATA was last updated.
+ *
+ *                                If EVENT_GEN is < priv->event_generation,
+ *                                or is 0, it means afr_inode_refresh() needs
+ *                                to be called to recalculate the bitmaps.
+ */
 
-        priv   = this->private;
+int
+__afr_inode_read_subvol_get_small (inode_t *inode, xlator_t *this,
+				   unsigned char *data, unsigned char *metadata,
+				   int *event_p)
+{
+	afr_private_t *priv = NULL;
+	int ret = -1;
+	uint16_t datamap = 0;
+	uint16_t metadatamap = 0;
+	uint32_t event = 0;
+	uint64_t val = 0;
+	int i = 0;
+
+	priv = this->private;
+
+	ret = __inode_ctx_get (inode, this, &val);
+	if (ret < 0)
+		return ret;
+
+	metadatamap = (val & 0x000000000000ffff);
+	datamap =     (val & 0x00000000ffff0000) >> 16;
+	event =       (val & 0xffffffff00000000) >> 32;
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (metadata)
+			metadata[i] = (metadatamap >> i) & 1;
+		if (data)
+			data[i] = (datamap >> i) & 1;
+	}
 
-        for (i = 0; i < priv->child_count; i++) {
-                ret = dict_set_uint64 (xattr_req, priv->pending_key[i],
-                                       3 * sizeof(int32_t));
-                if (ret < 0)
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "%s: Unable to set dict value for %s",
-                                path, priv->pending_key[i]);
-                /* 3 = data+metadata+entry */
-        }
-        ret = dict_set_int32 (xattr_req, GF_GFIDLESS_LOOKUP, 1);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG, "%s: failed to set gfidless "
-                        "lookup", path);
-        }
+	if (event_p)
+		*event_p = event;
+	return ret;
 }
 
+
 int
-afr_lookup_xattr_req_prepare (afr_local_t *local, xlator_t *this,
-                              dict_t *xattr_req, loc_t *loc, void **gfid_req)
-{
-        int     ret = -ENOMEM;
+__afr_inode_read_subvol_set_small (inode_t *inode, xlator_t *this,
+				   unsigned char *data, unsigned char *metadata,
+				   int event)
+{
+	afr_private_t *priv = NULL;
+	uint16_t datamap = 0;
+	uint16_t metadatamap = 0;
+	uint64_t val = 0;
+	int i = 0;
+
+	priv = this->private;
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (data[i])
+			datamap |= (1 << i);
+		if (metadata[i])
+			metadatamap |= (1 << i);
+	}
 
-        GF_ASSERT (gfid_req);
+	val = ((uint64_t) metadatamap) |
+		(((uint64_t) datamap) << 16) |
+		(((uint64_t) event) << 32);
 
-        *gfid_req = NULL;
-        local->xattr_req = dict_new ();
-        if (!local->xattr_req)
-                goto out;
-        if (xattr_req)
-                dict_copy (xattr_req, local->xattr_req);
+	return __inode_ctx_set (inode, this, &val);
+}
 
-        afr_xattr_req_prepare (this, local->xattr_req, loc->path);
-        ret = dict_set_uint64 (local->xattr_req, GLUSTERFS_INODELK_COUNT, 0);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: Unable to set dict value for %s",
-                        loc->path, GLUSTERFS_INODELK_COUNT);
-        }
-        ret = dict_set_uint64 (local->xattr_req, GLUSTERFS_ENTRYLK_COUNT, 0);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: Unable to set dict value for %s",
-                        loc->path, GLUSTERFS_ENTRYLK_COUNT);
-        }
 
-        ret = dict_set_uint32 (local->xattr_req, GLUSTERFS_PARENT_ENTRYLK, 0);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: Unable to set dict value for %s",
-                        loc->path, GLUSTERFS_PARENT_ENTRYLK);
-        }
+int
+__afr_inode_read_subvol_reset_small (inode_t *inode, xlator_t *this)
+{
+	int ret = -1;
+	uint16_t datamap = 0;
+	uint16_t metadatamap = 0;
+	uint32_t event = 0;
+	uint64_t val = 0;
 
-        ret = dict_get_ptr (local->xattr_req, "gfid-req", gfid_req);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "%s: failed to get the gfid from dict", loc->path);
-                *gfid_req = NULL;
-        } else {
-                if (loc->parent != NULL)
-                        dict_del (local->xattr_req, "gfid-req");
-        }
-        ret = 0;
-out:
-        return ret;
-}
+	ret = __inode_ctx_get (inode, this, &val);
+	(void) ret;
 
-void
-afr_lookup_save_gfid (uuid_t dst, void* new, const loc_t *loc)
-{
-        inode_t  *inode = NULL;
-
-        inode = loc->inode;
-        if (inode && !uuid_is_null (inode->gfid))
-                uuid_copy (dst, inode->gfid);
-        else if (!uuid_is_null (loc->gfid))
-                uuid_copy (dst, loc->gfid);
-        else if (new && !uuid_is_null (new))
-                uuid_copy (dst, new);
+	metadatamap = (val & 0x000000000000ffff) >> 0;
+	datamap =     (val & 0x00000000ffff0000) >> 16;
+	event = 0;
+
+	val = ((uint64_t) metadatamap) |
+		(((uint64_t) datamap) << 16) |
+		(((uint64_t) event) << 32);
+
+	return __inode_ctx_set (inode, this, &val);
 }
 
+
 int
-afr_errno_count (int32_t *children, int *child_errno,
-                 unsigned int child_count, int32_t op_errno)
+__afr_inode_read_subvol_get (inode_t *inode, xlator_t *this,
+			     unsigned char *data, unsigned char *metadata,
+			     int *event_p)
 {
-        int i = 0;
-        int errno_count = 0;
-        int child = 0;
+	afr_private_t *priv = NULL;
+	int ret = -1;
 
-        for (i = 0; i < child_count; i++) {
-                if (children) {
-                        child = children[i];
-                        if (child == -1)
-                                break;
-                } else {
-                        child = i;
-                }
-                if (child_errno[child] == op_errno)
-                        errno_count++;
-        }
-        return errno_count;
+	priv = this->private;
+
+	if (priv->child_count <= 16)
+		ret = __afr_inode_read_subvol_get_small (inode, this, data,
+							 metadata, event_p);
+	else
+		/* TBD: allocate structure with array and read from it */
+		ret = -1;
+
+	return ret;
 }
 
-int32_t
-afr_set_dict_gfid (dict_t *dict, uuid_t gfid)
+
+int
+__afr_inode_read_subvol_set (inode_t *inode, xlator_t *this, unsigned char *data,
+			     unsigned char *metadata, int event)
 {
-        int ret       = 0;
-        uuid_t *pgfid = NULL;
+	afr_private_t *priv = NULL;
+	int ret = -1;
 
-        GF_ASSERT (gfid);
+	priv = this->private;
 
-        pgfid = GF_CALLOC (1, sizeof (uuid_t), gf_common_mt_char);
-        if (!pgfid) {
-                ret = -1;
-                goto out;
-        }
+	if (priv->child_count <= 16)
+		ret = __afr_inode_read_subvol_set_small (inode, this, data,
+							 metadata, event);
+	else
+		ret = -1;
 
-        uuid_copy (*pgfid, gfid);
+	return ret;
+}
 
-        ret = dict_set_dynptr (dict, "gfid-req", pgfid, sizeof (uuid_t));
-        if (ret)
-                gf_log (THIS->name, GF_LOG_ERROR, "gfid set failed");
 
-out:
-        if (ret && pgfid)
-                GF_FREE (pgfid);
+int
+__afr_inode_read_subvol_reset (inode_t *inode, xlator_t *this)
+{
+	afr_private_t *priv = NULL;
+	int ret = -1;
 
-        return ret;
+	priv = this->private;
+
+	if (priv->child_count <= 16)
+		ret = __afr_inode_read_subvol_reset_small (inode, this);
+	else
+		ret = -1;
+
+	return ret;
 }
 
-afr_inode_ctx_t*
-afr_inode_ctx_get_from_addr (uint64_t addr, int32_t child_count)
+
+int
+afr_inode_read_subvol_get (inode_t *inode, xlator_t *this, unsigned char *data,
+			   unsigned char *metadata, int *event_p)
 {
-        int             ret  = -1;
-        afr_inode_ctx_t *ctx = NULL;
-        size_t          size = 0;
+	int ret = -1;
 
-        GF_ASSERT (child_count > 0);
+	LOCK(&inode->lock);
+	{
+		ret = __afr_inode_read_subvol_get (inode, this, data,
+						   metadata, event_p);
+	}
+	UNLOCK(&inode->lock);
 
-        if (!addr) {
-                ctx = GF_CALLOC (1, sizeof (*ctx),
-                                 gf_afr_mt_inode_ctx_t);
-                if (!ctx)
-                        goto out;
-                size = sizeof (*ctx->fresh_children);
-                ctx->fresh_children = GF_CALLOC (child_count, size,
-                                                 gf_afr_mt_int32_t);
-                if (!ctx->fresh_children)
-                        goto out;
-        } else {
-                ctx = (afr_inode_ctx_t*) (long) addr;
-        }
-        ret = 0;
-out:
-        if (ret && ctx) {
-                if (ctx->fresh_children)
-                        GF_FREE (ctx->fresh_children);
-                GF_FREE (ctx);
-                ctx = NULL;
-        }
-        return ctx;
+	return ret;
 }
 
-void
-afr_inode_get_ctx (xlator_t *this, inode_t *inode, afr_inode_params_t *params)
+
+int
+afr_inode_read_subvol_set (inode_t *inode, xlator_t *this, unsigned char *data,
+			   unsigned char *metadata, int event)
 {
-        GF_ASSERT (inode);
-        GF_ASSERT (params);
+	int ret = -1;
 
-        int             ret = 0;
-        afr_inode_ctx_t *ctx = NULL;
-        afr_private_t   *priv = NULL;
-        int             i = 0;
-        uint64_t        ctx_addr   = 0;
-        int32_t         read_child = -1;
-        int32_t         *fresh_children = NULL;
+	LOCK(&inode->lock);
+	{
+		ret = __afr_inode_read_subvol_set (inode, this, data, metadata,
+						   event);
+	}
+	UNLOCK(&inode->lock);
 
-        priv = this->private;
-        LOCK (&inode->lock);
-        {
-                ret = __inode_ctx_get (inode, this, &ctx_addr);
-                if (ret < 0)
-                        goto unlock;
-                ctx = afr_inode_ctx_get_from_addr (ctx_addr, priv->child_count);
-                if (!ctx)
-                        goto unlock;
-                switch (params->op) {
-                case AFR_INODE_GET_READ_CTX:
-                        fresh_children = params->u.read_ctx.children;
-                        read_child = (int32_t)(ctx->masks &
-                                               AFR_ICTX_READ_CHILD_MASK);
-                        params->u.read_ctx.read_child = read_child;
-                        if (!fresh_children)
-                                goto unlock;
-                        for (i = 0; i < priv->child_count; i++)
-                                fresh_children[i] = ctx->fresh_children[i];
-                        break;
-                case AFR_INODE_GET_OPENDIR_DONE:
-                        params->u.value = _gf_false;
-                        if (ctx->masks & AFR_ICTX_OPENDIR_DONE_MASK)
-                                params->u.value = _gf_true;
-                        break;
-                case AFR_INODE_GET_SPLIT_BRAIN:
-                        params->u.value = _gf_false;
-                        if (ctx->masks & AFR_ICTX_SPLIT_BRAIN_MASK)
-                                params->u.value = _gf_true;
-                        ;
-                        break;
-                default:
-                        GF_ASSERT (0);
-                        break;
-                }
-        }
-unlock:
-        UNLOCK (&inode->lock);
+	return ret;
 }
 
-gf_boolean_t
-afr_is_split_brain (xlator_t *this, inode_t *inode)
+
+int
+afr_inode_read_subvol_reset (inode_t *inode, xlator_t *this)
 {
-        afr_inode_params_t params = {0};
+	int ret = -1;
+
+	LOCK(&inode->lock);
+	{
+		ret = __afr_inode_read_subvol_reset (inode, this);
+	}
+	UNLOCK(&inode->lock);
 
-        params.op = AFR_INODE_GET_SPLIT_BRAIN;
-        afr_inode_get_ctx (this, inode, &params);
-        return params.u.value;
+	return ret;
 }
 
-gf_boolean_t
-afr_is_opendir_done (xlator_t *this, inode_t *inode)
-{
-        afr_inode_params_t params = {0};
 
-        params.op = AFR_INODE_GET_OPENDIR_DONE;
-        afr_inode_get_ctx (this, inode, &params);
-        return params.u.value;
+int
+afr_accused_fill (xlator_t *this, dict_t *xdata, unsigned char *accused,
+		  afr_transaction_type type)
+{
+	afr_private_t *priv = NULL;
+	int i = 0;
+	int idx = afr_index_for_transaction_type (type);
+	void *pending_raw = NULL;
+	int pending[3];
+	int ret = 0;
+
+	priv = this->private;
+
+	for (i = 0; i < priv->child_count; i++) {
+		ret = dict_get_ptr (xdata, priv->pending_key[i],
+				    &pending_raw);
+		if (ret) /* no pending flags */
+			continue;
+		memcpy (pending, pending_raw, sizeof(pending));
+
+		if (ntoh32 (pending[idx]))
+			accused[i] = 1;
+	}
+
+	return 0;
 }
 
 
-int32_t
-afr_inode_get_read_ctx (xlator_t *this, inode_t *inode, int32_t *fresh_children)
+int
+afr_accuse_smallfiles (xlator_t *this, struct afr_reply *replies,
+		       unsigned char *data_accused)
 {
-        afr_inode_params_t      params = {0};
+	int i = 0;
+	afr_private_t *priv = NULL;
+	uint64_t maxsize = 0;
+
+	priv = this->private;
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (data_accused[i])
+			continue;
+		if (replies[i].poststat.ia_size > maxsize)
+			maxsize = replies[i].poststat.ia_size;
+	}
 
-        params.op = AFR_INODE_GET_READ_CTX;
-        params.u.read_ctx.children = fresh_children;
-        afr_inode_get_ctx (this, inode, &params);
-        return params.u.read_ctx.read_child;
+	for (i = 0; i < priv->child_count; i++) {
+		if (data_accused[i])
+			continue;
+		if (replies[i].poststat.ia_size < maxsize)
+			data_accused[i] = 1;
+	}
+
+	return 0;
 }
 
-void
-afr_inode_ctx_set_read_child (afr_inode_ctx_t *ctx, int32_t read_child)
-{
-        uint64_t        remaining_mask = 0;
-        uint64_t        mask         = 0;
 
-        remaining_mask = (~AFR_ICTX_READ_CHILD_MASK & ctx->masks);
-        mask = (AFR_ICTX_READ_CHILD_MASK & read_child);
-        ctx->masks = remaining_mask | mask;
+int
+afr_replies_interpret (call_frame_t *frame, xlator_t *this, inode_t *inode)
+{
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	struct afr_reply *replies = NULL;
+	int event_generation = 0;
+	int i = 0;
+	unsigned char *data_accused = NULL;
+	unsigned char *metadata_accused = NULL;
+	unsigned char *data_readable = NULL;
+	unsigned char *metadata_readable = NULL;
+	int ret = 0;
+
+	local = frame->local;
+	priv = this->private;
+	replies = local->replies;
+	event_generation = local->event_generation;
+
+	data_accused = alloca0 (priv->child_count);
+	data_readable = alloca0 (priv->child_count);
+	metadata_accused = alloca0 (priv->child_count);
+	metadata_readable = alloca0 (priv->child_count);
+
+	for (i = 0; i < priv->child_count; i++) {
+		data_readable[i] = 1;
+		metadata_readable[i] = 1;
+	}
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!replies[i].valid) {
+			data_readable[i] = 0;
+			metadata_readable[i] = 0;
+			continue;
+		}
+
+		if (replies[i].op_ret == -1) {
+			data_readable[i] = 0;
+			metadata_readable[i] = 0;
+			continue;
+		}
+
+		afr_accused_fill (this, replies[i].xdata, data_accused,
+				  (inode->ia_type == IA_IFDIR) ?
+				   AFR_ENTRY_TRANSACTION : AFR_DATA_TRANSACTION);
+
+		afr_accused_fill (this, replies[i].xdata,
+				  metadata_accused, AFR_METADATA_TRANSACTION);
+
+	}
+
+	if (inode->ia_type != IA_IFDIR)
+		afr_accuse_smallfiles (this, replies, data_accused);
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (data_accused[i]) {
+			data_readable[i] = 0;
+			ret = 1;
+		}
+		if (metadata_accused[i]) {
+			metadata_readable[i] = 0;
+			ret = 1;
+		}
+	}
+
+	afr_inode_read_subvol_set (inode, this, data_readable,
+				   metadata_readable, event_generation);
+	return ret;
 }
 
-void
-afr_inode_ctx_set_read_ctx (afr_inode_ctx_t *ctx, int32_t read_child,
-                            int32_t *fresh_children, int32_t child_count)
+
+
+int
+afr_refresh_selfheal_done (int ret, call_frame_t *heal, void *opaque)
 {
-        int             i            = 0;
-
-        afr_inode_ctx_set_read_child (ctx, read_child);
-        for (i = 0; i < child_count; i++) {
-                if (fresh_children)
-                        ctx->fresh_children[i] = fresh_children[i];
-                else
-                        ctx->fresh_children[i] = -1;
-        }
+	if (heal)
+		STACK_DESTROY (heal->root);
+	return 0;
 }
 
-void
-afr_inode_ctx_rm_stale_children (afr_inode_ctx_t *ctx, int32_t *stale_children,
-                                 int32_t child_count)
+int
+afr_inode_refresh_err (call_frame_t *frame, xlator_t *this)
 {
-        int             i            = 0;
-        int32_t         read_child   = -1;
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	int i = 0;
+	int err = 0;
 
-        GF_ASSERT (stale_children);
-        for (i = 0; i < child_count; i++) {
-                if (stale_children[i] == -1)
-                        break;
-                afr_children_rm_child (ctx->fresh_children,
-                                       stale_children[i], child_count);
-        }
-        read_child = (int32_t)(ctx->masks & AFR_ICTX_READ_CHILD_MASK);
-        if (!afr_is_child_present (ctx->fresh_children, child_count,
-                                   read_child))
-                afr_inode_ctx_set_read_child (ctx, ctx->fresh_children[0]);
-}
+	local = frame->local;
+	priv = this->private;
 
-void
-afr_inode_ctx_set_opendir_done (afr_inode_ctx_t *ctx)
-{
-        uint64_t        remaining_mask = 0;
-        uint64_t        mask = 0;
+	for (i = 0; i < priv->child_count; i++) {
+		if (local->replies[i].valid && !local->replies[i].op_ret) {
+			err = 0;
+			goto ret;
+		}
+	}
 
-        remaining_mask = (~AFR_ICTX_OPENDIR_DONE_MASK & ctx->masks);
-        mask = (0xFFFFFFFFFFFFFFFFULL & AFR_ICTX_OPENDIR_DONE_MASK);
-        ctx->masks = remaining_mask | mask;
+	err = afr_final_errno (local, priv);
+ret:
+	return -err;
 }
 
-void
-afr_inode_ctx_set_splitbrain (afr_inode_ctx_t *ctx, gf_boolean_t set)
+
+int
+afr_refresh_selfheal_wrap (void *opaque)
 {
-        uint64_t        remaining_mask = 0;
-        uint64_t        mask = 0;
+	call_frame_t *frame = opaque;
+	afr_local_t *local = NULL;
+	xlator_t *this = NULL;
+	int err = 0;
 
-        if (set) {
-                remaining_mask = (~AFR_ICTX_SPLIT_BRAIN_MASK & ctx->masks);
-                mask = (0xFFFFFFFFFFFFFFFFULL & AFR_ICTX_SPLIT_BRAIN_MASK);
-                ctx->masks = remaining_mask | mask;
-        } else {
-                ctx->masks = (~AFR_ICTX_SPLIT_BRAIN_MASK & ctx->masks);
-        }
-}
+	local = frame->local;
+	this = frame->this;
 
-void
-afr_inode_set_ctx (xlator_t *this, inode_t *inode, afr_inode_params_t *params)
-{
-        GF_ASSERT (inode);
-        GF_ASSERT (params);
+	afr_selfheal (frame->this, local->refreshinode->gfid);
 
-        int             ret = 0;
-        afr_inode_ctx_t *ctx            = NULL;
-        afr_private_t   *priv           = NULL;
-        uint64_t        ctx_addr        = 0;
-        gf_boolean_t    set             = _gf_false;
-        int32_t         read_child      = -1;
-        int32_t         *fresh_children = NULL;
-        int32_t         *stale_children = NULL;
+	afr_selfheal_unlocked_discover (frame, local->refreshinode,
+					local->refreshinode->gfid,
+					local->replies);
 
-        priv = this->private;
-        LOCK (&inode->lock);
-        {
-                ret = __inode_ctx_get (inode, this, &ctx_addr);
-                if (ret < 0)
-                        ctx_addr = 0;
-                ctx = afr_inode_ctx_get_from_addr (ctx_addr, priv->child_count);
-                if (!ctx)
-                        goto unlock;
-                switch (params->op) {
-                case AFR_INODE_SET_READ_CTX:
-                        read_child = params->u.read_ctx.read_child;
-                        fresh_children = params->u.read_ctx.children;
-                        afr_inode_ctx_set_read_ctx (ctx, read_child,
-                                                    fresh_children,
-                                                    priv->child_count);
-                        break;
-                case AFR_INODE_RM_STALE_CHILDREN:
-                        stale_children = params->u.read_ctx.children;
-                        afr_inode_ctx_rm_stale_children (ctx,
-                                                         stale_children,
-                                                         priv->child_count);
-                        break;
-                case AFR_INODE_SET_OPENDIR_DONE:
-                        afr_inode_ctx_set_opendir_done (ctx);
-                        break;
-                case AFR_INODE_SET_SPLIT_BRAIN:
-                        set = params->u.value;
-                        afr_inode_ctx_set_splitbrain (ctx, set);
-                        break;
-                default:
-                        GF_ASSERT (0);
-                        break;
-                }
-                ret = __inode_ctx_put (inode, this, (uint64_t)ctx);
-                if (ret) {
-                        gf_log_callingfn (this->name, GF_LOG_ERROR, "failed to "
-                                          "set the inode ctx (%s)",
-                                          uuid_utoa (inode->gfid));
-                }
-        }
-unlock:
-        UNLOCK (&inode->lock);
-}
+	afr_replies_interpret (frame, this, local->refreshinode);
 
-void
-afr_set_split_brain (xlator_t *this, inode_t *inode, gf_boolean_t set)
-{
-        afr_inode_params_t      params = {0};
+	err = afr_inode_refresh_err (frame, this);
 
-        params.op = AFR_INODE_SET_SPLIT_BRAIN;
-        params.u.value          = set;
-        afr_inode_set_ctx (this, inode, &params);
-}
+	afr_replies_wipe (local, this->private);
 
-void
-afr_set_opendir_done (xlator_t *this, inode_t *inode)
-{
-        afr_inode_params_t params = {0};
+	local->refreshfn (frame, this, err);
 
-        params.op = AFR_INODE_SET_OPENDIR_DONE;
-        afr_inode_set_ctx (this, inode, &params);
+	return 0;
 }
 
-void
-afr_inode_set_read_ctx (xlator_t *this, inode_t *inode, int32_t read_child,
-                        int32_t *fresh_children)
+
+gf_boolean_t
+afr_selfheal_enabled (xlator_t *this)
 {
-        afr_inode_params_t params = {0};
-        afr_private_t      *priv  = NULL;
+	afr_private_t *priv = NULL;
+	gf_boolean_t data = _gf_false;
 
-        priv = this->private;
-        GF_ASSERT (read_child >= 0);
-        GF_ASSERT (fresh_children);
-        GF_ASSERT (afr_is_child_present (fresh_children, priv->child_count,
-                                         read_child));
-
-        params.op = AFR_INODE_SET_READ_CTX;
-        params.u.read_ctx.read_child     = read_child;
-        params.u.read_ctx.children = fresh_children;
-        afr_inode_set_ctx (this, inode, &params);
+	priv = this->private;
+
+	gf_string2boolean (priv->data_self_heal, &data);
+
+	return data || priv->metadata_self_heal || priv->entry_self_heal;
 }
 
-void
-afr_inode_rm_stale_children (xlator_t *this, inode_t *inode,
-                             int32_t *stale_children)
+
+
+int
+afr_inode_refresh_done (call_frame_t *frame, xlator_t *this)
 {
-        afr_inode_params_t params = {0};
+	call_frame_t *heal = NULL;
+	afr_local_t *local = NULL;
+	int ret = 0;
+	int err = 0;
 
-        GF_ASSERT (stale_children);
+	local = frame->local;
 
-        params.op = AFR_INODE_RM_STALE_CHILDREN;
-        params.u.read_ctx.children = stale_children;
-        afr_inode_set_ctx (this, inode, &params);
-}
+	ret = afr_replies_interpret (frame, this, local->refreshinode);
 
-gf_boolean_t
-afr_is_source_child (int32_t *sources, int32_t child_count, int32_t child)
-{
-        gf_boolean_t             source_xattrs = _gf_false;
+	err = afr_inode_refresh_err (frame, this);
 
-        GF_ASSERT (child < child_count);
+	afr_replies_wipe (local, this->private);
 
-        if ((child >= 0) && (child < child_count) &&
-             sources[child]) {
-                source_xattrs = _gf_true;
-        }
-        return source_xattrs;
+	if (ret && afr_selfheal_enabled (this)) {
+		heal = copy_frame (frame);
+		if (heal)
+			heal->root->pid = -1;
+		ret = synctask_new (this->ctx->env, afr_refresh_selfheal_wrap,
+				    afr_refresh_selfheal_done, heal, frame);
+		if (ret)
+			goto refresh_done;
+	} else {
+	refresh_done:
+		local->refreshfn (frame, this, err);
+	}
+
+	return 0;
 }
 
-gf_boolean_t
-afr_is_child_present (int32_t *success_children, int32_t child_count,
-                      int32_t child)
-{
-        gf_boolean_t             success_child = _gf_false;
-        int                      i = 0;
 
-        GF_ASSERT (child < child_count);
+int
+afr_inode_refresh_subvol_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+			      int op_ret, int op_errno, inode_t *inode,
+			      struct iatt *buf, dict_t *xdata, struct iatt *par)
+{
+	afr_local_t *local = NULL;
+	int call_child = (long) cookie;
+	int call_count = 0;
+
+	local = frame->local;
+
+	local->replies[call_child].valid = 1;
+	local->replies[call_child].op_ret = op_ret;
+	local->replies[call_child].op_errno = op_errno;
+	if (op_ret != -1) {
+		local->replies[call_child].poststat = *buf;
+		local->replies[call_child].postparent = *par;
+		local->replies[call_child].xdata = dict_ref (xdata);
+	}
+
+	call_count = afr_frame_return (frame);
 
-        for (i = 0; i < child_count; i++) {
-                if (success_children[i] == -1)
-                        break;
-                if (child == success_children[i]) {
-                        success_child = _gf_true;
-                        break;
-                }
-        }
-        return success_child;
+	if (call_count == 0)
+		afr_inode_refresh_done (frame, this);
+
+	return 0;
 }
 
-gf_boolean_t
-afr_is_read_child (int32_t *success_children, int32_t *sources,
-                   int32_t child_count, int32_t child)
+
+int
+afr_inode_refresh_subvol (call_frame_t *frame, xlator_t *this, int i,
+			  inode_t *inode, dict_t *xdata)
 {
-        gf_boolean_t             success_child = _gf_false;
-        gf_boolean_t             source        = _gf_false;
+	loc_t loc = {0, };
+	afr_private_t *priv = NULL;
 
-        GF_ASSERT (success_children);
-        GF_ASSERT (child_count > 0);
+	priv = this->private;
 
-        success_child = afr_is_child_present (success_children, child_count,
-                                              child);
-        if (!success_child)
-                goto out;
-        if (NULL == sources) {
-                source = _gf_true;
-                goto out;
-        }
-        source = afr_is_source_child (sources, child_count, child);
-out:
-        return (success_child && source);
+	loc.inode = inode;
+	uuid_copy (loc.gfid, inode->gfid);
+
+	STACK_WIND_COOKIE (frame, afr_inode_refresh_subvol_cbk,
+			   (void *) (long) i, priv->children[i],
+			   priv->children[i]->fops->lookup, &loc, xdata);
+	return 0;
 }
 
-/* If sources is NULL the xattrs are assumed to be of source for all
- * success_children.
- */
+
 int
-afr_select_read_child_from_policy (int32_t *success_children, int32_t child_count,
-                                   int32_t prev_read_child,
-                                   int32_t config_read_child, int32_t *sources)
+afr_inode_refresh_do (call_frame_t *frame, xlator_t *this)
 {
-        int32_t                  read_child   = -1;
-        int                      i            = 0;
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	int call_count = 0;
+	int i = 0;
+	dict_t *xdata = NULL;
 
-        GF_ASSERT (success_children);
+	priv = this->private;
+	local = frame->local;
 
-        read_child = prev_read_child;
-        if (afr_is_read_child (success_children, sources, child_count,
-                               read_child))
-                goto out;
+	afr_replies_wipe (local, priv);
 
-        read_child = config_read_child;
-        if (afr_is_read_child (success_children, sources, child_count,
-                               read_child))
-                goto out;
+	xdata = dict_new ();
+	if (!xdata) {
+		afr_inode_refresh_done (frame, this);
+		return 0;
+	}
 
-        for (i = 0; i < child_count; i++) {
-                read_child = success_children[i];
-                if (read_child < 0)
-                        break;
-                if (afr_is_read_child (success_children, sources, child_count,
-                                       read_child))
-                        goto out;
-        }
-        read_child = -1;
+	if (afr_xattr_req_prepare (this, xdata) != 0) {
+		dict_unref (xdata);
+		afr_inode_refresh_done (frame, this);
+		return 0;
+	}
 
-out:
-        return read_child;
+	local->call_count = AFR_COUNT (local->child_up, priv->child_count);
+
+	call_count = local->call_count;
+	for (i = 0; i < priv->child_count; i++) {
+		if (!local->child_up[i])
+			continue;
+
+		afr_inode_refresh_subvol (frame, this, i, local->refreshinode,
+					  xdata);
+
+		if (!--call_count)
+			break;
+	}
+
+	dict_unref (xdata);
+
+	return 0;
 }
 
-/* This function should be used when all the success_children are sources
- */
-void
-afr_set_read_ctx_from_policy (xlator_t *this, inode_t *inode,
-                              int32_t *fresh_children, int32_t prev_read_child,
-                              int32_t config_read_child)
+
+int
+afr_inode_refresh (call_frame_t *frame, xlator_t *this, inode_t *inode,
+		   afr_inode_refresh_cbk_t refreshfn)
 {
-        int                      read_child = -1;
-        afr_private_t            *priv = NULL;
+	afr_local_t *local = NULL;
 
-        priv = this->private;
-        read_child = afr_select_read_child_from_policy (fresh_children,
-                                                        priv->child_count,
-                                                        prev_read_child,
-                                                        config_read_child,
-                                                        NULL);
-        if (read_child >= 0)
-                afr_inode_set_read_ctx (this, inode, read_child,
-                                        fresh_children);
+	local = frame->local;
+
+	local->refreshfn = refreshfn;
+
+	if (local->refreshinode) {
+		inode_unref (local->refreshinode);
+		local->refreshinode = NULL;
+	}
+
+	local->refreshinode = inode_ref (inode);
+
+	afr_inode_refresh_do (frame, this);
+
+	return 0;
 }
 
-/* afr_next_call_child ()
- * This is a common function used by all the read-type fops
- * This function should not be called with the inode's read_children array.
- * The fop's handler should make a copy of the inode's read_children,
- * preferred read_child into the local vars, because while this function is
- * in execution there is a chance for inode's read_ctx to change.
- */
-int32_t
-afr_next_call_child (int32_t *fresh_children, unsigned char *child_up,
-                     size_t child_count, int32_t *last_index,
-                     int32_t read_child)
+
+int
+afr_xattr_req_prepare (xlator_t *this, dict_t *xattr_req)
 {
-        int             next_index      = 0;
-        int32_t         next_call_child = -1;
+        int             i           = 0;
+        afr_private_t   *priv       = NULL;
+        int             ret         = 0;
+
+        priv   = this->private;
 
-        GF_ASSERT (last_index);
+        for (i = 0; i < priv->child_count; i++) {
+                ret = dict_set_uint64 (xattr_req, priv->pending_key[i],
+                                       AFR_NUM_CHANGE_LOGS * sizeof(int));
+                if (ret < 0)
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "Unable to set dict value for %s",
+                                priv->pending_key[i]);
+                /* 3 = data+metadata+entry */
+        }
+        ret = dict_set_uint64 (xattr_req, AFR_DIRTY,
+			       AFR_NUM_CHANGE_LOGS * sizeof(int));
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG, "failed to set dirty "
+                        "query flag");
+        }
 
-        next_index = *last_index;
-retry:
-        next_index++;
-        if ((next_index >= child_count) ||
-           (fresh_children[next_index] == -1))
-                goto out;
-        if ((fresh_children[next_index] == read_child) ||
-           (!child_up[fresh_children[next_index]]))
-                goto retry;
-        *last_index = next_index;
-        next_call_child = fresh_children[next_index];
-out:
-        return next_call_child;
+	return ret;
 }
 
- /* This function should not be called with the inode's read_children array.
- * The fop's handler should make a copy of the inode's read_children,
- * preferred read_child into the local vars, because while this function is
- * in execution there is a chance for inode's read_ctx to change.
- */
-int32_t
-afr_get_call_child (xlator_t *this, unsigned char *child_up, int32_t read_child,
-                    int32_t *fresh_children,
-                    int32_t *call_child, int32_t *last_index)
+int
+afr_lookup_xattr_req_prepare (afr_local_t *local, xlator_t *this,
+                              dict_t *xattr_req, loc_t *loc)
 {
-        int             ret   = 0;
-        afr_private_t   *priv = NULL;
-        int             i     = 0;
-
-        GF_ASSERT (child_up);
-        GF_ASSERT (call_child);
-        GF_ASSERT (last_index);
-        GF_ASSERT (fresh_children);
+        int     ret = -ENOMEM;
 
-        if (read_child < 0) {
-                ret = -EIO;
+        local->xattr_req = dict_new ();
+        if (!local->xattr_req)
                 goto out;
-        }
-        priv = this->private;
-        *call_child = -1;
-        *last_index = -1;
+        if (xattr_req)
+                dict_copy (xattr_req, local->xattr_req);
 
-        if (child_up[read_child]) {
-                *call_child = read_child;
-        } else {
-                for (i = 0; i < priv->child_count; i++) {
-                        if (fresh_children[i] == -1)
-                                break;
-                        if (child_up[fresh_children[i]]) {
-                                *call_child = fresh_children[i];
-                                ret = 0;
-                                break;
-                        }
-                }
+        ret = afr_xattr_req_prepare (this, local->xattr_req);
+	if (ret < 0) {
+		gf_log (this->name, GF_LOG_WARNING,
+			"%s: Unable to prepare xattr_req", loc->path);
+	}
 
-                if (*call_child == -1) {
-                        ret = -ENOTCONN;
-                        goto out;
-                }
+        ret = dict_set_uint64 (local->xattr_req, GLUSTERFS_INODELK_COUNT, 0);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: Unable to set dict value for %s",
+                        loc->path, GLUSTERFS_INODELK_COUNT);
+        }
+        ret = dict_set_uint64 (local->xattr_req, GLUSTERFS_ENTRYLK_COUNT, 0);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: Unable to set dict value for %s",
+                        loc->path, GLUSTERFS_ENTRYLK_COUNT);
+        }
 
-                *last_index = i;
+        ret = dict_set_uint32 (local->xattr_req, GLUSTERFS_PARENT_ENTRYLK, 0);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: Unable to set dict value for %s",
+                        loc->path, GLUSTERFS_PARENT_ENTRYLK);
         }
+
+        ret = 0;
 out:
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d, call_child: %d, "
-                "last_index: %d", ret, *call_child, *last_index);
         return ret;
 }
 
-void
-afr_reset_xattr (dict_t **xattr, unsigned int child_count)
+
+int
+afr_hash_child (inode_t *inode, int32_t child_count, int hashmode)
 {
-        unsigned int i = 0;
+        uuid_t gfid_copy = {0,};
+        pid_t pid;
 
-        if (!xattr)
-                goto out;
-        for (i = 0; i < child_count; i++) {
-                if (xattr[i]) {
-                        dict_unref (xattr[i]);
-                        xattr[i] = NULL;
-                }
+        if (!hashmode) {
+                return -1;
         }
-out:
-        return;
+
+        if (inode) {
+               uuid_copy (gfid_copy, inode->gfid);
+        }
+
+        if (hashmode > 1) {
+                /*
+                 * Why getpid?  Because it's one of the cheapest calls
+                 * available - faster than gethostname etc. - and returns a
+                 * constant-length value that's sure to be shorter than a UUID.
+                 * It's still very unlikely to be the same across clients, so
+                 * it still provides good mixing.  We're not trying for
+                 * perfection here.  All we need is a low probability that
+                 * multiple clients won't converge on the same subvolume.
+                 */
+                pid = getpid();
+                memcpy (gfid_copy, &pid, sizeof(pid));
+        }
+
+        return SuperFastHash((char *)gfid_copy,
+                             sizeof(gfid_copy)) % child_count;
 }
 
-void
-afr_local_sh_cleanup (afr_local_t *local, xlator_t *this)
-{
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
 
-        sh = &local->self_heal;
-        priv = this->private;
+int
+afr_read_subvol_select_by_policy (inode_t *inode, xlator_t *this,
+				  unsigned char *readable)
+{
+	afr_private_t *priv = NULL;
+	int read_subvol = -1;
+	int i = 0;
 
-        if (sh->buf)
-                GF_FREE (sh->buf);
+	priv = this->private;
 
-        if (sh->parentbufs)
-                GF_FREE (sh->parentbufs);
+	/* first preference - explicitly specified or local subvolume */
+	if (priv->read_child >= 0 && readable[priv->read_child])
+		return priv->read_child;
 
-        if (sh->inode)
-                inode_unref (sh->inode);
+	/* second preference - use hashed mode */
+	read_subvol = afr_hash_child (inode, priv->child_count,
+				      priv->hash_mode);
+	if (read_subvol >= 0 && readable[read_subvol])
+		return read_subvol;
 
-        if (sh->xattr) {
-                afr_reset_xattr (sh->xattr, priv->child_count);
-                GF_FREE (sh->xattr);
-        }
+	for (i = 0; i < priv->child_count; i++) {
+		if (readable[i])
+			return i;
+	}
 
-        if (sh->child_errno)
-                GF_FREE (sh->child_errno);
+	/* no readable subvolumes, either split brain or all subvols down */
 
-        afr_matrix_cleanup (sh->pending_matrix, priv->child_count);
-        afr_matrix_cleanup (sh->delta_matrix, priv->child_count);
+	return -1;
+}
 
-        if (sh->sources)
-                GF_FREE (sh->sources);
 
-        if (sh->success)
-                GF_FREE (sh->success);
+int
+afr_inode_read_subvol_type_get (inode_t *inode, xlator_t *this,
+				unsigned char *readable, int *event_p,
+				int type)
+{
+	int ret = -1;
 
-        if (sh->locked_nodes)
-                GF_FREE (sh->locked_nodes);
+	if (type == AFR_METADATA_TRANSACTION)
+		ret = afr_inode_read_subvol_get (inode, this, 0, readable,
+						 event_p);
+	else
+		ret = afr_inode_read_subvol_get (inode, this, readable, 0,
+						 event_p);
+	return ret;
+}
 
-        if (sh->healing_fd) {
-                fd_unref (sh->healing_fd);
-                sh->healing_fd = NULL;
-        }
 
-        if (sh->linkname)
-                GF_FREE ((char *)sh->linkname);
+int
+afr_read_subvol_get (inode_t *inode, xlator_t *this, int *subvol_p,
+		     int *event_p, afr_transaction_type type)
+{
+	afr_private_t *priv = NULL;
+	unsigned char *data_readable = NULL;
+	unsigned char *metadata_readable = NULL;
+	unsigned char *readable = NULL;
+	unsigned char *intersection = NULL;
+	int subvol = -1;
+	int event = 0;
 
-        if (sh->success_children)
-                GF_FREE (sh->success_children);
+	priv = this->private;
 
-        if (sh->fresh_children)
-                GF_FREE (sh->fresh_children);
+	readable = alloca0 (priv->child_count);
+	data_readable = alloca0 (priv->child_count);
+	metadata_readable = alloca0 (priv->child_count);
+	intersection = alloca0 (priv->child_count);
 
-        if (sh->fresh_parent_dirs)
-                GF_FREE (sh->fresh_parent_dirs);
+	afr_inode_read_subvol_type_get (inode, this, readable, &event, type);
 
-        loc_wipe (&sh->parent_loc);
-        loc_wipe (&sh->lookup_loc);
+	afr_inode_read_subvol_get (inode, this, data_readable, metadata_readable,
+				   &event);
 
-        if (sh->checksum)
-                GF_FREE (sh->checksum);
+	AFR_INTERSECT (intersection, data_readable, metadata_readable,
+		       priv->child_count);
 
-        if (sh->write_needed)
-                GF_FREE (sh->write_needed);
-        if (sh->healing_fd)
-                fd_unref (sh->healing_fd);
+	if (AFR_COUNT (intersection, priv->child_count) > 0)
+		subvol = afr_read_subvol_select_by_policy (inode, this,
+							   intersection);
+	else
+		subvol = afr_read_subvol_select_by_policy (inode, this,
+							   readable);
+	if (subvol_p)
+		*subvol_p = subvol;
+	if (event_p)
+		*event_p = event;
+	return subvol;
 }
 
 
 void
 afr_local_transaction_cleanup (afr_local_t *local, xlator_t *this)
 {
-        afr_private_t * priv = NULL;
+        afr_private_t *priv    = NULL;
+        int           i        = 0;
 
         priv = this->private;
 
         afr_matrix_cleanup (local->pending, priv->child_count);
 
-        if (local->internal_lock.locked_nodes)
-                GF_FREE (local->internal_lock.locked_nodes);
+        GF_FREE (local->internal_lock.locked_nodes);
 
-        if (local->internal_lock.inode_locked_nodes)
-                GF_FREE (local->internal_lock.inode_locked_nodes);
-
-        if (local->internal_lock.entry_locked_nodes)
-                GF_FREE (local->internal_lock.entry_locked_nodes);
+        for (i = 0; local->internal_lock.inodelk[i].domain; i++) {
+                GF_FREE (local->internal_lock.inodelk[i].locked_nodes);
+        }
 
-        if (local->internal_lock.lower_locked_nodes)
-                GF_FREE (local->internal_lock.lower_locked_nodes);
+        GF_FREE (local->internal_lock.lower_locked_nodes);
 
+        afr_entry_lockee_cleanup (&local->internal_lock);
 
         GF_FREE (local->transaction.pre_op);
-        GF_FREE (local->transaction.child_errno);
         GF_FREE (local->transaction.eager_lock);
 
         GF_FREE (local->transaction.basename);
@@ -821,8 +867,37 @@ afr_local_transaction_cleanup (afr_local_t *local, xlator_t *this)
 
         loc_wipe (&local->transaction.parent_loc);
         loc_wipe (&local->transaction.new_parent_loc);
+
+}
+
+
+void
+afr_replies_wipe (afr_local_t *local, afr_private_t *priv)
+{
+	int i;
+
+	if (!local->replies)
+		return;
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (local->replies[i].xdata) {
+			dict_unref (local->replies[i].xdata);
+			local->replies[i].xdata = NULL;
+		}
+	}
+
+	memset (local->replies, 0, sizeof(*local->replies) * priv->child_count);
 }
 
+void
+afr_remove_eager_lock_stub (afr_local_t *local)
+{
+        LOCK (&local->fd->lock);
+        {
+                list_del_init (&local->transaction.eager_locked);
+        }
+        UNLOCK (&local->fd->lock);
+}
 
 void
 afr_local_cleanup (afr_local_t *local, xlator_t *this)
@@ -832,7 +907,11 @@ afr_local_cleanup (afr_local_t *local, xlator_t *this)
         if (!local)
                 return;
 
-        afr_local_sh_cleanup (local, this);
+	syncbarrier_destroy (&local->barrier);
+
+        if (local->transaction.eager_lock_on &&
+            !list_empty (&local->transaction.eager_locked))
+                afr_remove_eager_lock_stub (local);
 
         afr_local_transaction_cleanup (local, this);
 
@@ -850,55 +929,33 @@ afr_local_cleanup (afr_local_t *local, xlator_t *this)
         if (local->dict)
                 dict_unref (local->dict);
 
-        if (local->child_up)
-                GF_FREE (local->child_up);
-
-        if (local->child_errno)
-                GF_FREE (local->child_errno);
-
-        if (local->fresh_children)
-                GF_FREE (local->fresh_children);
+	afr_replies_wipe (local, priv);
+	GF_FREE(local->replies);
 
-        if (local->fd_open_on)
-                GF_FREE (local->fd_open_on);
-
-        { /* lookup */
-                if (local->cont.lookup.xattrs) {
-                        afr_reset_xattr (local->cont.lookup.xattrs,
-                                         priv->child_count);
-                        GF_FREE (local->cont.lookup.xattrs);
-                        local->cont.lookup.xattrs = NULL;
-                }
+        GF_FREE (local->child_up);
 
-                if (local->cont.lookup.xattr) {
-                        dict_unref (local->cont.lookup.xattr);
-                }
+        GF_FREE (local->read_attempted);
 
-                if (local->cont.lookup.inode) {
-                        inode_unref (local->cont.lookup.inode);
-                }
+        GF_FREE (local->readable);
 
-                if (local->cont.lookup.postparents)
-                        GF_FREE (local->cont.lookup.postparents);
+	if (local->inode)
+		inode_unref (local->inode);
 
-                if (local->cont.lookup.bufs)
-                        GF_FREE (local->cont.lookup.bufs);
+	if (local->parent)
+		inode_unref (local->parent);
 
-                if (local->cont.lookup.success_children)
-                        GF_FREE (local->cont.lookup.success_children);
+	if (local->parent2)
+		inode_unref (local->parent2);
 
-                if (local->cont.lookup.sources)
-                        GF_FREE (local->cont.lookup.sources);
-        }
+	if (local->refreshinode)
+		inode_unref (local->refreshinode);
 
         { /* getxattr */
-                if (local->cont.getxattr.name)
-                        GF_FREE (local->cont.getxattr.name);
+                GF_FREE (local->cont.getxattr.name);
         }
 
         { /* lk */
-                if (local->cont.lk.locked_nodes)
-                        GF_FREE (local->cont.lk.locked_nodes);
+                GF_FREE (local->cont.lk.locked_nodes);
         }
 
         { /* create */
@@ -953,8 +1010,7 @@ afr_local_cleanup (afr_local_t *local, xlator_t *this)
         }
 
         { /* opendir */
-                if (local->cont.opendir.checksum)
-                        GF_FREE (local->cont.opendir.checksum);
+                GF_FREE (local->cont.opendir.checksum);
         }
 
         { /* readdirp */
@@ -987,1254 +1043,730 @@ afr_frame_return (call_frame_t *frame)
         return call_count;
 }
 
-int
-afr_set_elem_count_get (unsigned char *elems, int child_count)
-{
-        int i   = 0;
-        int ret = 0;
-
-        for (i = 0; i < child_count; i++)
-                if (elems[i])
-                        ret++;
-        return ret;
-}
-
-/**
- * up_children_count - return the number of children that are up
- */
-
-unsigned int
-afr_up_children_count (unsigned char *child_up, unsigned int child_count)
-{
-        return afr_set_elem_count_get (child_up, child_count);
-}
-
-unsigned int
-afr_locked_children_count (unsigned char *children, unsigned int child_count)
-{
-        return afr_set_elem_count_get (children, child_count);
-}
-
-unsigned int
-afr_pre_op_done_children_count (unsigned char *pre_op,
-                                unsigned int child_count)
-{
-        return afr_set_elem_count_get (pre_op, child_count);
-}
 
 gf_boolean_t
-afr_is_fresh_lookup (loc_t *loc, xlator_t *this)
-{
-        uint64_t          ctx = 0;
-        int32_t           ret = 0;
-
-        GF_ASSERT (loc);
-        GF_ASSERT (this);
-        GF_ASSERT (loc->inode);
-
-        ret = inode_ctx_get (loc->inode, this, &ctx);
-        if (0 == ret)
-                return _gf_false;
-        return _gf_true;
-}
-
-void
-afr_update_loc_gfids (loc_t *loc, struct iatt *buf, struct iatt *postparent)
-{
-        GF_ASSERT (loc);
-        GF_ASSERT (buf);
+afr_is_entry_possibly_under_txn (afr_local_t *local, xlator_t *this)
+{
+	int i = 0;
+	int tmp = 0;
+	afr_private_t *priv = NULL;
+
+	priv = this->private;
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!local->replies[i].xdata)
+			continue;
+		if (dict_get_int32 (local->replies[i].xdata,
+				    GLUSTERFS_PARENT_ENTRYLK,
+				    &tmp) == 0)
+			if (tmp)
+				return _gf_true;
+	}
 
-        uuid_copy (loc->gfid, buf->ia_gfid);
-        if (postparent)
-                uuid_copy (loc->pargfid, postparent->ia_gfid);
+	return _gf_false;
 }
 
-int
-afr_lookup_build_response_params (afr_local_t *local, xlator_t *this)
-{
-        struct iatt     *buf = NULL;
-        struct iatt     *postparent = NULL;
-        dict_t          **xattr = NULL;
-        int32_t         *success_children = NULL;
-        int32_t         *sources = NULL;
-        afr_private_t   *priv = NULL;
-        int32_t         read_child = -1;
-        int             ret = 0;
-        int             i = 0;
-
-        GF_ASSERT (local);
-
-        buf = &local->cont.lookup.buf;
-        postparent = &local->cont.lookup.postparent;
-        xattr = &local->cont.lookup.xattr;
-        priv = this->private;
 
-        read_child = afr_inode_get_read_ctx (this, local->cont.lookup.inode,
-                                             local->fresh_children);
-        if (read_child < 0) {
-                ret = -1;
-                goto out;
-        }
-        success_children = local->cont.lookup.success_children;
-        sources = local->cont.lookup.sources;
-        memset (sources, 0, sizeof (*sources) * priv->child_count);
-        afr_children_intersection_get (local->fresh_children, success_children,
-                                       sources, priv->child_count);
-        if (!sources[read_child]) {
-                read_child = -1;
-                for (i = 0; i < priv->child_count; i++) {
-                        if (sources[i]) {
-                                read_child = i;
-                                break;
-                        }
-                }
-        }
-        if (read_child < 0) {
-                ret = -1;
-                goto out;
-        }
-        gf_log (this->name, GF_LOG_DEBUG, "Building lookup response from %d",
-                read_child);
-        if (!*xattr)
-                *xattr = dict_ref (local->cont.lookup.xattrs[read_child]);
-        *buf = local->cont.lookup.bufs[read_child];
-        *postparent = local->cont.lookup.postparents[read_child];
-
-        if (IA_INVAL == local->cont.lookup.inode->ia_type) {
-                /* fix for RT #602 */
-                local->cont.lookup.inode->ia_type = buf->ia_type;
-        }
-out:
-        return ret;
-}
+/*
+ * Quota size xattrs are not maintained by afr. There is a
+ * possibility that they differ even when both the directory changelog xattrs
+ * suggest everything is fine. So if there is at least one 'source' check among
+ * the sources which has the maximum quota size. Otherwise check among all the
+ * available ones for maximum quota size. This way if there is a source and
+ * stale copies it always votes for the 'source'.
+ * */
 
 static void
-afr_lookup_update_lk_counts (afr_local_t *local, xlator_t *this,
-                            int child_index, dict_t *xattr)
-{
-        uint32_t inodelk_count = 0;
-        uint32_t entrylk_count = 0;
-        int      ret           = -1;
-        uint32_t parent_entrylk = 0;
-
-        GF_ASSERT (local);
-        GF_ASSERT (this);
-        GF_ASSERT (xattr);
-        GF_ASSERT (child_index >= 0);
-
-        ret = dict_get_uint32 (xattr, GLUSTERFS_INODELK_COUNT,
-                               &inodelk_count);
-        if (ret == 0)
-                local->inodelk_count += inodelk_count;
+afr_handle_quota_size (call_frame_t *frame, xlator_t *this)
+{
+	unsigned char *readable = NULL;
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	struct afr_reply *replies = NULL;
+	int i = 0;
+	uint64_t size = 0;
+	uint64_t max_size = 0;
+	int readable_cnt = 0;
+
+	local = frame->local;
+	priv = this->private;
+	replies = local->replies;
+
+	readable = alloca0 (priv->child_count);
+
+	afr_inode_read_subvol_get (local->inode, this, readable, 0, 0);
+
+	readable_cnt = AFR_COUNT (readable, priv->child_count);
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!replies[i].valid || replies[i].op_ret == -1)
+			continue;
+		if (readable_cnt && !readable[i])
+			continue;
+		if (!replies[i].xdata)
+			continue;
+		if (dict_get_uint64 (replies[i].xdata, QUOTA_SIZE_KEY, &size))
+			continue;
+		if (size > max_size)
+			max_size = size;
+	}
 
-        ret = dict_get_uint32 (xattr, GLUSTERFS_ENTRYLK_COUNT,
-                               &entrylk_count);
-        if (ret == 0)
-                local->entrylk_count += entrylk_count;
-        ret = dict_get_uint32 (xattr, GLUSTERFS_PARENT_ENTRYLK,
-                               &parent_entrylk);
-        if (!ret)
-                local->cont.lookup.parent_entrylk += parent_entrylk;
+	if (!max_size)
+		return;
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!replies[i].valid || replies[i].op_ret == -1)
+			continue;
+		if (readable_cnt && !readable[i])
+			continue;
+		if (!replies[i].xdata)
+			continue;
+		if (dict_set_uint64 (replies[i].xdata, QUOTA_SIZE_KEY, max_size))
+			continue;
+	}
 }
 
+
 static void
-afr_lookup_set_self_heal_params_by_xattr (afr_local_t *local, xlator_t *this,
-                                          dict_t *xattr)
+afr_lookup_done (call_frame_t *frame, xlator_t *this)
 {
-        GF_ASSERT (local);
-        GF_ASSERT (this);
-        GF_ASSERT (xattr);
+        afr_private_t       *priv  = NULL;
+        afr_local_t         *local = NULL;
+	int                 i = -1;
+	int                 op_errno = 0;
+	int                 read_subvol = 0;
+	unsigned char      *readable = NULL;
+	int                 event = 0;
+	struct afr_reply   *replies = NULL;
+	uuid_t              read_gfid = {0, };
+	gf_boolean_t        locked_entry = _gf_false;
+	gf_boolean_t        can_interpret = _gf_true;
 
-        if (afr_sh_has_metadata_pending (xattr, this)) {
-                local->self_heal.do_metadata_self_heal = _gf_true;
-                gf_log(this->name, GF_LOG_DEBUG,
-                       "metadata self-heal is pending for %s.",
-                       local->loc.path);
-        }
+        priv  = this->private;
+        local = frame->local;
+	replies = local->replies;
 
-        if (afr_sh_has_entry_pending (xattr, this)) {
-                local->self_heal.do_entry_self_heal = _gf_true;
-                gf_log(this->name, GF_LOG_DEBUG,
-                       "entry self-heal is pending for %s.", local->loc.path);
-        }
+	locked_entry = afr_is_entry_possibly_under_txn (local, this);
 
-        if (afr_sh_has_data_pending (xattr, this)) {
-                local->self_heal.do_data_self_heal = _gf_true;
-                gf_log(this->name, GF_LOG_DEBUG,
-                       "data self-heal is pending for %s.", local->loc.path);
-        }
-}
+	readable = alloca0 (priv->child_count);
 
-static void
-afr_detect_self_heal_by_iatt (afr_local_t *local, xlator_t *this,
-                            struct iatt *buf, struct iatt *lookup_buf)
-{
-        if (PERMISSION_DIFFERS (buf, lookup_buf)) {
-                /* mismatching permissions */
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "permissions differ for %s ", local->loc.path);
-                local->self_heal.do_metadata_self_heal = _gf_true;
-        }
+	afr_inode_read_subvol_get (local->loc.parent, this, readable,
+				   NULL, &event);
 
-        if (OWNERSHIP_DIFFERS (buf, lookup_buf)) {
-                /* mismatching permissions */
-                local->self_heal.do_metadata_self_heal = _gf_true;
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "ownership differs for %s ", local->loc.path);
-        }
+	/* First, check if we have an ESTALE from somewhere,
+	   If so, propagate that so that a revalidate can be
+	   issued
+	*/
+	op_errno = afr_final_errno (frame->local, this->private);
+	local->op_errno = op_errno;
+        if (op_errno == ESTALE) {
+		local->op_errno = op_errno;
+		local->op_ret = -1;
+                goto unwind;
+	}
 
-        if (SIZE_DIFFERS (buf, lookup_buf)
-            && IA_ISREG (buf->ia_type)) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "size differs for %s ", local->loc.path);
-                local->self_heal.do_data_self_heal = _gf_true;
-        }
+	read_subvol = -1;
+	for (i = 0; i < priv->child_count; i++) {
+		if (!replies[i].valid)
+			continue;
+
+		if (locked_entry && replies[i].op_ret == -1 &&
+		    replies[i].op_errno == ENOENT) {
+			/* Second, check entry is still
+			   "underway" in creation */
+			local->op_ret = -1;
+			local->op_errno = ENOENT;
+			read_subvol = i;
+			goto unwind;
+		}
+
+		if (replies[i].op_ret == -1)
+			continue;
+
+		if (read_subvol == -1 || !readable[read_subvol]) {
+			read_subvol = i;
+			uuid_copy (read_gfid, replies[i].poststat.ia_gfid);
+			local->op_ret = 0;
+		}
+	}
 
-        if (uuid_compare (buf->ia_gfid, lookup_buf->ia_gfid)) {
-                /* mismatching gfid */
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: gfid different on subvolume", local->loc.path);
-        }
-}
+	if (read_subvol == -1)
+		goto unwind;
+	/* We now have a read_subvol, which is readable[] (if there
+	   were any). Next we look for GFID mismatches. We don't
+	   consider a GFID mismatch as an error if read_subvol is
+	   readable[] but the mismatching GFID subvol is not.
+	*/
+	for (i = 0; i < priv->child_count; i++) {
+		if (!replies[i].valid || replies[i].op_ret == -1) {
+			if (priv->child_up[i])
+				can_interpret = _gf_false;
+			continue;
+		}
+
+		if (!uuid_compare (replies[i].poststat.ia_gfid,
+				   read_gfid))
+			continue;
+
+		can_interpret = _gf_false;
+
+		if (locked_entry)
+			continue;
+
+		/* Now GFIDs mismatch. It's OK as long as this subvol
+		   is not readable[] but read_subvol is */
+		if (readable[read_subvol] && !readable[i])
+			continue;
+
+		/* LOG ERROR */
+		local->op_ret = -1;
+		local->op_errno = EIO;
+		goto unwind;
+	}
 
-static void
-afr_detect_self_heal_by_lookup_status (afr_local_t *local, xlator_t *this,
-                                       gf_boolean_t split_brain)
-{
-        GF_ASSERT (local);
-        GF_ASSERT (this);
+	/* Forth, for the finalized GFID, pick the best subvolume
+	   to return stats from.
+	*/
+	if (can_interpret) {
+		/* It is safe to call afr_replies_interpret() because we have
+		   a response from all the UP subvolumes and all of them resolved
+		   to the same GFID
+		*/
+		if (afr_replies_interpret (frame, this, local->inode)) {
+			read_subvol = afr_data_subvol_get (local->inode, this,
+							   0, 0);
+			afr_inode_read_subvol_reset (local->inode, this);
+			goto cant_interpret;
+		} else {
+			read_subvol = afr_data_subvol_get (local->inode, this,
+							   0, 0);
+		}
+	} else {
+	cant_interpret:
+		if (read_subvol == -1)
+			dict_del (replies[0].xdata, GF_CONTENT_KEY);
+		else
+			dict_del (replies[read_subvol].xdata, GF_CONTENT_KEY);
+	}
 
-        if ((local->success_count > 0) && (local->enoent_count > 0)) {
-                local->self_heal.do_metadata_self_heal = _gf_true;
-                local->self_heal.do_data_self_heal     = _gf_true;
-                local->self_heal.do_entry_self_heal    = _gf_true;
-                local->self_heal.do_gfid_self_heal    = _gf_true;
-                local->self_heal.do_missing_entry_self_heal    = _gf_true;
-                gf_log(this->name, GF_LOG_DEBUG,
-                       "entries are missing in lookup of %s.",
-                       local->loc.path);
-                //If all self-heals are needed no need to check for other rules
-                goto out;
-        }
+	afr_handle_quota_size (frame, this);
 
-        if ((local->success_count > 0) && split_brain &&
-            IA_ISREG (local->cont.lookup.inode->ia_type)) {
-                local->self_heal.do_data_self_heal = _gf_true;
-                local->self_heal.do_metadata_self_heal = _gf_true;
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "split brain detected during lookup of %s.",
-                        local->loc.path);
-        }
+unwind:
+	if (read_subvol == -1)
+		read_subvol = 0;
 
-out:
-        return;
+	AFR_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
+			  local->inode, &local->replies[read_subvol].poststat,
+			  local->replies[read_subvol].xdata,
+			  &local->replies[read_subvol].postparent);
 }
 
-gf_boolean_t
-afr_can_self_heal_proceed (afr_self_heal_t *sh, afr_private_t *priv)
+/*
+ * During a lookup, some errors are more "important" than
+ * others in that they must be given higher priority while
+ * returning to the user.
+ *
+ * The hierarchy is ESTALE > ENOENT > others
+ */
+
+int
+afr_higher_errno (int32_t old_errno, int32_t new_errno)
 {
-        GF_ASSERT (sh);
-        GF_ASSERT (priv);
+	if (old_errno == ENODATA || new_errno == ENODATA)
+		return ENODATA;
+	if (old_errno == ESTALE || new_errno == ESTALE)
+		return ESTALE;
+	if (old_errno == ENOENT || new_errno == ENOENT)
+		return ENOENT;
 
-        return (sh->do_gfid_self_heal
-                || sh->do_missing_entry_self_heal
-                || (afr_data_self_heal_enabled (priv->data_self_heal) &&
-                    sh->do_data_self_heal)
-                || (priv->metadata_self_heal && sh->do_metadata_self_heal)
-                || (priv->entry_self_heal && sh->do_entry_self_heal));
+	return new_errno;
 }
 
-afr_transaction_type
-afr_transaction_type_get (ia_type_t ia_type)
-{
-        afr_transaction_type    type = AFR_METADATA_TRANSACTION;
 
-        GF_ASSERT (ia_type != IA_INVAL);
+int
+afr_final_errno (afr_local_t *local, afr_private_t *priv)
+{
+	int i = 0;
+	int op_errno = 0;
+	int tmp_errno = 0;
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!local->replies[i].valid)
+			continue;
+		if (local->replies[i].op_ret == 0)
+			continue;
+		tmp_errno = local->replies[i].op_errno;
+		op_errno = afr_higher_errno (op_errno, tmp_errno);
+	}
 
-        if (IA_ISDIR (ia_type)) {
-                type = AFR_ENTRY_TRANSACTION;
-        } else if (IA_ISREG (ia_type)) {
-                type = AFR_DATA_TRANSACTION;
-        }
-        return type;
+	return op_errno;
 }
 
-int
-afr_lookup_select_read_child (afr_local_t *local, xlator_t *this,
-                              int32_t *read_child)
+static int
+get_pathinfo_host (char *pathinfo, char *hostname, size_t size)
 {
-        ia_type_t               ia_type        = IA_INVAL;
-        int32_t                 source         = -1;
-        int                     ret            = -1;
-        dict_t                  **xattrs       = NULL;
-        int32_t                 *success_children = NULL;
-        afr_transaction_type    type           = AFR_METADATA_TRANSACTION;
-
-        GF_ASSERT (local);
-        GF_ASSERT (this);
-        GF_ASSERT (local->success_count > 0);
+        char    *start = NULL;
+        char    *end = NULL;
+        int     ret  = -1;
+        int     i    = 0;
 
-        success_children = local->cont.lookup.success_children;
-        /*We can take the success_children[0] only because we already
-         *handle the conflicting children other wise, we could select the
-         *read_child based on wrong file type
-         */
-        ia_type = local->cont.lookup.bufs[success_children[0]].ia_type;
-        type = afr_transaction_type_get (ia_type);
-        xattrs = local->cont.lookup.xattrs;
-        source = afr_lookup_select_read_child_by_txn_type (this, local, xattrs,
-                                                           type);
-        if (source < 0) {
-                gf_log (this->name, GF_LOG_DEBUG, "failed to select source "
-                        "for %s", local->loc.path);
+        if (!pathinfo)
+                goto out;
+
+        start = strchr (pathinfo, ':');
+        if (!start)
+                goto out;
+        end = strrchr (pathinfo, ':');
+        if (start == end)
                 goto out;
-        }
 
-        gf_log (this->name, GF_LOG_DEBUG, "Source selected as %d for %s",
-                source, local->loc.path);
-        *read_child = source;
+        memset (hostname, 0, size);
+        i = 0;
+        while (++start != end)
+                hostname[i++] = *start;
         ret = 0;
 out:
         return ret;
 }
 
-static inline gf_boolean_t
-afr_is_transaction_running (afr_local_t *local)
-{
-        GF_ASSERT (local->fop == GF_FOP_LOOKUP);
-        return ((local->inodelk_count > 0) || (local->entrylk_count > 0));
-}
-
-void
-afr_launch_self_heal (call_frame_t *frame, xlator_t *this, inode_t *inode,
-                      gf_boolean_t background, ia_type_t ia_type, char *reason,
-                      void (*gfid_sh_success_cbk) (call_frame_t *sh_frame,
-                                                   xlator_t *this),
-                      int (*unwind) (call_frame_t *frame, xlator_t *this,
-                                     int32_t op_ret, int32_t op_errno,
-                                     int32_t sh_failed))
-{
-        afr_local_t             *local = NULL;
-        char                    sh_type_str[256] = {0,};
-        char                    *bg = "";
-
-        GF_ASSERT (frame);
-        GF_ASSERT (this);
-        GF_ASSERT (inode);
-        GF_ASSERT (ia_type != IA_INVAL);
-
-        local = frame->local;
-        local->self_heal.background = background;
-        local->self_heal.type       = ia_type;
-        local->self_heal.unwind     = unwind;
-        local->self_heal.gfid_sh_success_cbk     = gfid_sh_success_cbk;
-
-        afr_self_heal_type_str_get (&local->self_heal,
-                                    sh_type_str,
-                                    sizeof (sh_type_str));
-
-        if (background)
-                bg = "background";
-        gf_log (this->name, GF_LOG_DEBUG,
-                "%s %s self-heal triggered. path: %s, reason: %s", bg,
-                sh_type_str, local->loc.path, reason);
-
-        afr_self_heal (frame, this, inode);
-}
-
-unsigned int
-afr_gfid_missing_count (const char *xlator_name, int32_t *success_children,
-                        struct iatt *bufs, unsigned int child_count,
-                        const char *path)
+int
+afr_local_pathinfo (char *pathinfo, gf_boolean_t *local)
 {
-        unsigned int    gfid_miss_count   = 0;
-        int             i              = 0;
-        struct iatt     *child1        = NULL;
+        int             ret   = 0;
+        char            pathinfohost[1024] = {0};
+        char            localhost[1024] = {0};
+        xlator_t        *this = THIS;
 
-        for (i = 0; i < child_count; i++) {
-                if (success_children[i] == -1)
-                        break;
-                child1 = &bufs[success_children[i]];
-                if (uuid_is_null (child1->ia_gfid)) {
-                        gf_log (xlator_name, GF_LOG_DEBUG, "%s: gfid is null"
-                                " on subvolume %d", path, success_children[i]);
-                        gfid_miss_count++;
-                }
+        *local = _gf_false;
+        ret = get_pathinfo_host (pathinfo, pathinfohost, sizeof (pathinfohost));
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Invalid pathinfo: %s",
+                        pathinfo);
+                goto out;
         }
 
-        return gfid_miss_count;
-}
-
-static int
-afr_lookup_gfid_missing_count (afr_local_t *local, xlator_t *this)
-{
-        int32_t         *success_children = NULL;
-        afr_private_t   *priv          = NULL;
-        struct iatt     *bufs          = NULL;
-        int             miss_count     = 0;
-
-        priv = this->private;
-        bufs = local->cont.lookup.bufs;
-        success_children = local->cont.lookup.success_children;
-
-        miss_count =  afr_gfid_missing_count (this->name, success_children,
-                                              bufs, priv->child_count,
-                                              local->loc.path);
-        return miss_count;
-}
-
-gf_boolean_t
-afr_conflicting_iattrs (struct iatt *bufs, int32_t *success_children,
-                        unsigned int child_count, const char *path,
-                        const char *xlator_name)
-{
-        gf_boolean_t    conflicting    = _gf_false;
-        int             i              = 0;
-        struct iatt     *child1        = NULL;
-        struct iatt     *child2        = NULL;
-        uuid_t          *gfid          = NULL;
-
-        for (i = 0; i < child_count; i++) {
-                if (success_children[i] == -1)
-                        break;
-                child1 = &bufs[success_children[i]];
-                if ((!gfid) && (!uuid_is_null (child1->ia_gfid)))
-                        gfid = &child1->ia_gfid;
-
-                if (i == 0)
-                        continue;
-
-                child2 = &bufs[success_children[i-1]];
-                if (FILETYPE_DIFFERS (child1, child2)) {
-                        gf_log (xlator_name, GF_LOG_WARNING, "%s: filetype "
-                                "differs on subvolumes (%d, %d)", path,
-                                success_children[i-1], success_children[i]);
-                        conflicting = _gf_true;
-                        goto out;
-                }
-                if (!gfid || uuid_is_null (child1->ia_gfid))
-                        continue;
-                if (uuid_compare (*gfid, child1->ia_gfid)) {
-                       gf_log (xlator_name, GF_LOG_WARNING, "%s: gfid differs"
-                               " on subvolume %d", path, success_children[i]);
-                       conflicting = _gf_true;
-                       goto out;
-                }
+        ret = gethostname (localhost, sizeof (localhost));
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "gethostname() failed, "
+                        "reason: %s", strerror (errno));
+                goto out;
         }
-out:
-        return conflicting;
-}
-
-/* afr_update_gfid_from_iatts: This function should be called only if the
- * iatts are not conflicting.
- */
-void
-afr_update_gfid_from_iatts (uuid_t uuid, struct iatt *bufs,
-                            int32_t *success_children, unsigned int child_count)
-{
-        uuid_t          *gfid = NULL;
-        int             i = 0;
-        int             child = 0;
 
-        for (i = 0; i < child_count; i++) {
-                child = success_children[i];
-                if (child == -1)
-                        break;
-                if ((!gfid) && (!uuid_is_null (bufs[child].ia_gfid))) {
-                        gfid = &bufs[child].ia_gfid;
-                } else if (gfid && (!uuid_is_null (bufs[child].ia_gfid))) {
-                        if (uuid_compare (*gfid, bufs[child].ia_gfid)) {
-                                GF_ASSERT (0);
-                                goto out;
-                        }
-                }
-        }
-        if (gfid && (!uuid_is_null (*gfid)))
-                uuid_copy (uuid, *gfid);
+        if (!strcmp (localhost, pathinfohost))
+                *local = _gf_true;
 out:
-        return;
-}
-
-static gf_boolean_t
-afr_lookup_conflicting_entries (afr_local_t *local, xlator_t *this)
-{
-        afr_private_t           *priv = NULL;
-        gf_boolean_t            conflict = _gf_false;
-
-        priv = this->private;
-        conflict =  afr_conflicting_iattrs (local->cont.lookup.bufs,
-                                            local->cont.lookup.success_children,
-                                            priv->child_count, local->loc.path,
-                                            this->name);
-        return conflict;
-}
-
-gf_boolean_t
-afr_open_only_data_self_heal (char *data_self_heal)
-{
-        return !strcmp (data_self_heal, "open");
+        return ret;
 }
 
-gf_boolean_t
-afr_data_self_heal_enabled (char *data_self_heal)
+static int32_t
+afr_local_discovery_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+			 int32_t op_ret, int32_t op_errno, dict_t *dict,
+			 dict_t *xdata)
 {
-        gf_boolean_t    enabled = _gf_false;
+        int              ret            = 0;
+        char            *pathinfo       = NULL;
+        gf_boolean_t     is_local        = _gf_false;
+        afr_private_t   *priv           = NULL;
+        int32_t          child_index    = -1;
 
-        if (gf_string2boolean (data_self_heal, &enabled) == -1) {
-                enabled = !strcmp (data_self_heal, "open");
-                GF_ASSERT (enabled);
+        if (op_ret != 0) {
+                goto out;
         }
 
-        return enabled;
-}
-
-static void
-afr_lookup_set_self_heal_params (afr_local_t *local, xlator_t *this)
-{
-        int                     i = 0;
-        struct iatt             *bufs = NULL;
-        dict_t                  **xattr = NULL;
-        afr_private_t           *priv = NULL;
-        int32_t                 child1 = -1;
-        int32_t                 child2 = -1;
-        afr_self_heal_t         *sh = NULL;
-        gf_boolean_t            split_brain = _gf_false;
-
-        priv  = this->private;
-        sh = &local->self_heal;
-
-        split_brain = afr_is_split_brain (this, local->cont.lookup.inode);
-        split_brain = split_brain || local->cont.lookup.possible_spb;
-        afr_detect_self_heal_by_lookup_status (local, this, split_brain);
-
-        if (afr_lookup_gfid_missing_count (local, this))
-                local->self_heal.do_gfid_self_heal    = _gf_true;
-
-        if (_gf_true == afr_lookup_conflicting_entries (local, this))
-                local->self_heal.do_missing_entry_self_heal    = _gf_true;
-        else
-                afr_update_gfid_from_iatts (local->self_heal.sh_gfid_req,
-                                            local->cont.lookup.bufs,
-                                            local->cont.lookup.success_children,
-                                            priv->child_count);
-
-        bufs = local->cont.lookup.bufs;
-        for (i = 1; i < local->success_count; i++) {
-                child1 = local->cont.lookup.success_children[i-1];
-                child2 = local->cont.lookup.success_children[i];
-                afr_detect_self_heal_by_iatt (local, this,
-                                              &bufs[child1], &bufs[child2]);
-        }
+	priv = this->private;
+	child_index = (int32_t)(long)cookie;
 
-        xattr = local->cont.lookup.xattrs;
-        for (i = 0; i < local->success_count; i++) {
-                child1 = local->cont.lookup.success_children[i];
-                afr_lookup_set_self_heal_params_by_xattr (local, this,
-                                                          xattr[child1]);
+        ret = dict_get_str (dict, GF_XATTR_PATHINFO_KEY, &pathinfo);
+        if (ret != 0) {
+                goto out;
         }
-        if (afr_open_only_data_self_heal (priv->data_self_heal)
-            && !split_brain)
-                sh->do_data_self_heal = _gf_false;
-}
-
-int
-afr_self_heal_lookup_unwind (call_frame_t *frame, xlator_t *this,
-                             int32_t op_ret, int32_t op_errno,
-                             int32_t sh_failed)
-{
-        afr_local_t *local = NULL;
-        int         ret    = -1;
-        dict_t      *xattr = NULL;
-
-        local = frame->local;
-
-        if (op_ret == -1) {
-                local->op_ret = -1;
-                if (afr_error_more_important (local->op_errno, op_errno))
-                        local->op_errno = op_errno;
 
+        ret = afr_local_pathinfo (pathinfo, &is_local);
+        if (ret) {
                 goto out;
-        } else {
-                local->op_ret = 0;
         }
 
-        afr_lookup_done_success_action (frame, this, _gf_true);
-        xattr = local->cont.lookup.xattr;
-        if (xattr) {
-                ret = dict_set_int32 (xattr, "sh-failed", sh_failed);
-                if (ret)
-                        gf_log (this->name, GF_LOG_ERROR, "%s: Failed to set "
-                                "sh-failed to %d", local->loc.path, sh_failed);
+        /*
+         * Note that one local subvolume will override another here.  The only
+         * way to avoid that would be to retain extra information about whether
+         * the previous read_child is local, and it's just not worth it.  Even
+         * the slowest local subvolume is far preferable to a remote one.
+         */
+        if (is_local) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "selecting local read_child %s",
+                        priv->children[child_index]->name);
+                priv->read_child = child_index;
         }
 out:
-        AFR_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
-                          local->cont.lookup.inode, &local->cont.lookup.buf,
-                          local->cont.lookup.xattr,
-                          &local->cont.lookup.postparent);
-
+        STACK_DESTROY(frame->root);
         return 0;
 }
 
-//TODO: At the moment only lookup needs this, so not doing any checks, in the
-// future we will have to do fop specific operations
-void
-afr_post_gfid_sh_success (call_frame_t *sh_frame, xlator_t *this)
+static void
+afr_attempt_local_discovery (xlator_t *this, int32_t child_index)
 {
-        afr_local_t             *local = NULL;
-        afr_local_t             *sh_local = NULL;
-        afr_private_t           *priv = NULL;
-        afr_self_heal_t         *sh = NULL;
-        int                     i = 0;
-        struct iatt             *lookup_bufs = NULL;
-        struct iatt             *lookup_parentbufs = NULL;
-
-        sh_local = sh_frame->local;
-        sh       = &sh_local->self_heal;
-        local = sh->orig_frame->local;
-        lookup_bufs = local->cont.lookup.bufs;
-        lookup_parentbufs = local->cont.lookup.postparents;
-        priv = this->private;
-
-        memcpy (lookup_bufs, sh->buf, priv->child_count * sizeof (*sh->buf));
-        memcpy (lookup_parentbufs, sh->parentbufs,
-                priv->child_count * sizeof (*sh->parentbufs));
-
-        afr_reset_xattr (local->cont.lookup.xattrs, priv->child_count);
-        if (local->cont.lookup.xattr) {
-                dict_unref (local->cont.lookup.xattr);
-                local->cont.lookup.xattr = NULL;
-        }
+        call_frame_t    *newframe = NULL;
+        loc_t            tmploc = {0,};
+        afr_private_t   *priv = this->private;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (sh->xattr[i])
-                        local->cont.lookup.xattrs[i] = dict_ref (sh->xattr[i]);
+        newframe = create_frame(this,this->ctx->pool);
+        if (!newframe) {
+                return;
         }
 
-        afr_reset_children (local->cont.lookup.success_children,
-                            priv->child_count);
-        afr_children_copy (local->cont.lookup.success_children,
-                           sh->fresh_children, priv->child_count);
+        tmploc.gfid[sizeof(tmploc.gfid)-1] = 1;
+        STACK_WIND_COOKIE (newframe, afr_local_discovery_cbk,
+                           (void *)(long)child_index,
+                           priv->children[child_index],
+                           priv->children[child_index]->fops->getxattr,
+                           &tmploc, GF_XATTR_PATHINFO_KEY, NULL);
 }
 
-static void
-afr_lookup_perform_self_heal (call_frame_t *frame, xlator_t *this,
-                              gf_boolean_t *sh_launched)
+
+int
+afr_lookup_selfheal_wrap (void *opaque)
 {
-        unsigned int         up_count = 0;
-        afr_private_t       *priv    = NULL;
-        afr_local_t         *local   = NULL;
-        char                *reason  = NULL;
-
-        GF_ASSERT (sh_launched);
-        *sh_launched = _gf_false;
-        priv         = this->private;
-        local        = frame->local;
-
-        up_count  = afr_up_children_count (local->child_up, priv->child_count);
-        if (up_count == 1) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Only 1 child up - do not attempt to detect self heal");
-                goto out;
-        }
+	call_frame_t *frame = opaque;
+	afr_local_t *local = NULL;
+	xlator_t *this = NULL;
+	inode_t *inode = NULL;
 
-        afr_lookup_set_self_heal_params (local, this);
-        if (afr_can_self_heal_proceed (&local->self_heal, priv)) {
-                if  (afr_is_transaction_running (local))
-                        goto out;
+	local = frame->local;
+	this = frame->this;
 
-                reason = "lookup detected pending operations";
-                afr_launch_self_heal (frame, this, local->cont.lookup.inode,
-                                      _gf_true, local->cont.lookup.buf.ia_type,
-                                      reason, afr_post_gfid_sh_success,
-                                      afr_self_heal_lookup_unwind);
-                *sh_launched = _gf_true;
-        }
-out:
-        return;
-}
+	afr_selfheal_name (frame->this, local->loc.pargfid, local->loc.name);
 
-void
-afr_get_fresh_children (int32_t *success_children, int32_t *sources,
-                        int32_t *fresh_children, unsigned int child_count)
-{
-        unsigned int i = 0;
-        unsigned int j = 0;
+	afr_replies_wipe (local, this->private);
 
-        GF_ASSERT (success_children);
-        GF_ASSERT (sources);
-        GF_ASSERT (fresh_children);
+	inode = afr_selfheal_unlocked_lookup_on (frame, local->loc.parent,
+						 local->loc.name, local->replies,
+						 local->child_up);
+	if (inode)
+		inode_unref (inode);
+	afr_lookup_done (frame, this);
 
-        afr_reset_children (fresh_children, child_count);
-        for (i = 0; i < child_count; i++) {
-                if (success_children[i] == -1)
-                        break;
-                if (afr_is_read_child (success_children, sources, child_count,
-                                       success_children[i])) {
-                        fresh_children[j] = success_children[i];
-                        j++;
-                }
-        }
+	return 0;
 }
 
-static int
-afr_lookup_set_read_ctx (afr_local_t *local, xlator_t *this, int32_t read_child)
-{
-        afr_private_t           *priv = NULL;
 
-        GF_ASSERT (read_child >= 0);
+int
+afr_lookup_entry_heal (call_frame_t *frame, xlator_t *this)
+{
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	call_frame_t *heal = NULL;
+	int i = 0, first = -1;
+	gf_boolean_t need_heal = _gf_false;
+	struct afr_reply *replies = NULL;
+	int ret = 0;
+
+	local = frame->local;
+	replies = local->replies;
+	priv = this->private;
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!replies[i].valid)
+			continue;
+
+		if (first == -1) {
+			first = i;
+			continue;
+		}
+
+		if (replies[i].op_ret != replies[first].op_ret) {
+			need_heal = _gf_true;
+			break;
+		}
+
+		if (uuid_compare (replies[i].poststat.ia_gfid,
+				  replies[first].poststat.ia_gfid)) {
+			need_heal = _gf_true;
+			break;
+		}
+	}
 
-        priv = this->private;
-        afr_get_fresh_children (local->cont.lookup.success_children,
-                                local->cont.lookup.sources,
-                                local->fresh_children, priv->child_count);
-        afr_inode_set_read_ctx (this, local->cont.lookup.inode, read_child,
-                                local->fresh_children);
+	if (need_heal) {
+		heal = copy_frame (frame);
+		if (heal)
+			heal->root->pid = -1;
+		ret = synctask_new (this->ctx->env, afr_lookup_selfheal_wrap,
+				    afr_refresh_selfheal_done, heal, frame);
+		if (ret)
+			goto lookup_done;
+	} else {
+	lookup_done:
+		afr_lookup_done (frame, this);
+	}
 
-        return 0;
+	return ret;
 }
 
+
 int
-afr_lookup_done_success_action (call_frame_t *frame, xlator_t *this,
-                                gf_boolean_t fail_conflict)
+afr_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		int op_ret, int op_errno, inode_t *inode, struct iatt *buf,
+		dict_t *xdata, struct iatt *postparent)
 {
-        int32_t             read_child = -1;
-        int32_t             ret        = -1;
-        afr_local_t         *local     = NULL;
-        gf_boolean_t        fresh_lookup = _gf_false;
-
-        local   = frame->local;
-        fresh_lookup = local->cont.lookup.fresh_lookup;
-
-        if (local->loc.parent == NULL)
-                fail_conflict = _gf_true;
-
-        if (afr_lookup_conflicting_entries (local, this)) {
-                if (fail_conflict == _gf_false)
-                        ret = 0;
-                goto out;
-        }
-
-        ret = afr_lookup_select_read_child (local, this, &read_child);
-        if (!afr_is_transaction_running (local) || fresh_lookup) {
-                if (read_child < 0)
-                        goto out;
-
-                ret = afr_lookup_set_read_ctx (local, this, read_child);
-                if (ret)
-                        goto out;
-        }
+        afr_local_t *   local = NULL;
+        int             call_count      = -1;
+        int             child_index     = -1;
 
-        ret = afr_lookup_build_response_params (local, this);
-        if (ret)
-                goto out;
-        afr_update_loc_gfids (&local->loc,
-                              &local->cont.lookup.buf,
-                              &local->cont.lookup.postparent);
+	child_index = (long) cookie;
 
-        ret = 0;
-out:
-        if (ret) {
-                local->op_ret = -1;
-                local->op_errno = EIO;
-        }
-        return ret;
-}
+	local = frame->local;
 
-int
-afr_lookup_get_latest_subvol (afr_local_t *local, xlator_t *this)
-{
-        afr_private_t *priv = NULL;
-        int32_t       *success_children = NULL;
-        struct iatt   *bufs = NULL;
-        int           i = 0;
-        int           child = 0;
-        int           lsubvol = -1;
+	local->replies[child_index].valid = 1;
+	local->replies[child_index].op_ret = op_ret;
+	local->replies[child_index].op_errno = op_errno;
+	if (op_ret != -1) {
+		local->replies[child_index].poststat = *buf;
+		local->replies[child_index].postparent = *postparent;
+		if (xdata)
+			local->replies[child_index].xdata = dict_ref (xdata);
+	}
 
-        priv = this->private;
-        success_children = local->cont.lookup.success_children;
-        bufs = local->cont.lookup.bufs;
-        for (i = 0; i < priv->child_count; i++) {
-                child = success_children[i];
-                if (child == -1)
-                        break;
-                if (uuid_is_null (bufs[child].ia_gfid))
-                        continue;
-                if (lsubvol < 0) {
-                        lsubvol = child;
-                } else if (bufs[lsubvol].ia_ctime < bufs[child].ia_ctime) {
-                        lsubvol = child;
-                } else if ((bufs[lsubvol].ia_ctime == bufs[child].ia_ctime) &&
-                  (bufs[lsubvol].ia_ctime_nsec < bufs[child].ia_ctime_nsec)) {
-                        lsubvol = child;
-                }
+        call_count = afr_frame_return (frame);
+        if (call_count == 0) {
+		afr_lookup_entry_heal (frame, this);
         }
-        return lsubvol;
-}
-
-void
-afr_lookup_mark_other_entries_stale (afr_local_t *local, xlator_t *this,
-                                     int subvol)
-{
-        afr_private_t *priv = NULL;
-        int32_t       *success_children = NULL;
-        struct iatt   *bufs = NULL;
-        int           i = 0;
-        int           child = 0;
 
-        priv = this->private;
-        success_children = local->cont.lookup.success_children;
-        bufs = local->cont.lookup.bufs;
-        memcpy (local->fresh_children, success_children,
-                sizeof (*success_children) * priv->child_count);
-        for (i = 0; i < priv->child_count; i++) {
-                child = local->fresh_children[i];
-                if (child == -1)
-                        break;
-                if (child == subvol)
-                        continue;
-                if (uuid_is_null (bufs[child].ia_gfid) &&
-                    (bufs[child].ia_type == bufs[subvol].ia_type))
-                        continue;
-                afr_children_rm_child (success_children, child,
-                                       priv->child_count);
-                local->success_count--;
-        }
-        afr_reset_children (local->fresh_children, priv->child_count);
+	return 0;
 }
 
-void
-afr_succeed_lookup_on_latest_iatt (afr_local_t *local, xlator_t *this)
-{
-        int    lsubvol = 0;
-
-        if (!afr_lookup_conflicting_entries (local, this))
-                goto out;
-
-        lsubvol = afr_lookup_get_latest_subvol (local, this);
-        if (lsubvol < 0)
-                goto out;
-        afr_lookup_mark_other_entries_stale (local, this, lsubvol);
-out:
-        return;
-}
-
-gf_boolean_t
-afr_is_entry_possibly_under_creation (afr_local_t *local, xlator_t *this)
-{
-        /*
-         * We need to perform this test in lookup done and treat on going
-         * create/DELETE as ENOENT.
-         * Reason:
-        Multiple clients A, B and C are attempting 'mkdir -p /mnt/a/b/c'
-
-        1 Client A is in the middle of mkdir(/a). It has acquired lock.
-          It has performed mkdir(/a) on one subvol, and second one is still
-          in progress
-        2 Client B performs a lookup, sees directory /a on one,
-          ENOENT on the other, succeeds lookup.
-        3 Client B performs lookup on /a/b on both subvols, both return ENOENT
-          (one subvol because /a/b does not exist, another because /a
-          itself does not exist)
-        4 Client B proceeds to mkdir /a/b. It obtains entrylk on inode=/a with
-          basename=b on one subvol, but fails on other subvol as /a is yet to
-          be created by Client A.
-        5 Client A finishes mkdir of /a on other subvol
-        6 Client C also attempts to create /a/b, lookup returns ENOENT on
-          both subvols.
-        7 Client C tries to obtain entrylk on on inode=/a with basename=b,
-          obtains on one subvol (where B had failed), and waits for B to unlock
-          on other subvol.
-        8 Client B finishes mkdir() on one subvol with GFID-1 and completes
-          transaction and unlocks
-        9 Client C gets the lock on the second subvol, At this stage second
-          subvol already has /a/b created from Client B, but Client C does not
-          check that in the middle of mkdir transaction
-        10 Client C attempts mkdir /a/b on both subvols. It succeeds on
-           ONLY ONE (where Client B could not get lock because of
-           missing parent /a dir) with GFID-2, and gets EEXIST from ONE subvol.
-        This way we have /a/b in GFID mismatch. One subvol got GFID-1 because
-        Client B performed transaction on only one subvol (because entrylk()
-        could not be obtained on second subvol because of missing parent dir --
-        caused by premature/speculative succeeding of lookup() on /a when locks
-        are detected). Other subvol gets GFID-2 from Client C because while
-        it was waiting for entrylk() on both subvols, Client B was in the
-        middle of creating mkdir() on only one subvol, and Client C does not
-        "expect" this when it is between lock() and pre-op()/op() phase of the
-        transaction.
-         */
-	if (local->cont.lookup.parent_entrylk && local->enoent_count)
-		return _gf_true;
-
-	return _gf_false;
-}
 
 
 static void
-afr_lookup_done (call_frame_t *frame, xlator_t *this)
+afr_discover_done (call_frame_t *frame, xlator_t *this)
 {
-        int                 unwind = 1;
         afr_private_t       *priv  = NULL;
         afr_local_t         *local = NULL;
-        int                 ret = -1;
-        gf_boolean_t        sh_launched = _gf_false;
-        gf_boolean_t        fail_conflict = _gf_false;
-        int                 gfid_miss_count = 0;
-        int                 enotconn_count = 0;
-        int                 up_children_count = 0;
+	int                 i = -1;
+	int                 op_errno = 0;
+	int                 read_subvol = 0;
 
         priv  = this->private;
         local = frame->local;
 
-	if (afr_is_entry_possibly_under_creation (local, this)) {
-		local->op_ret = -1;
-		local->op_errno = ENOENT;
-		goto unwind;
+	for (i = 0; i < priv->child_count; i++) {
+		if (!local->replies[i].valid)
+			continue;
+		if (local->replies[i].op_ret == 0)
+			local->op_ret = 0;
 	}
 
-        if (local->op_ret < 0)
-                goto unwind;
+	op_errno = afr_final_errno (frame->local, this->private);
 
-        if (local->cont.lookup.parent_entrylk && local->success_count > 1)
-                afr_succeed_lookup_on_latest_iatt (local, this);
-
-        gfid_miss_count = afr_lookup_gfid_missing_count (local, this);
-        up_children_count = afr_up_children_count (local->child_up,
-                                                   priv->child_count);
-        enotconn_count = priv->child_count - up_children_count;
-        if ((gfid_miss_count == local->success_count) &&
-            (enotconn_count > 0)) {
-                local->op_ret = -1;
-                local->op_errno = EIO;
-                gf_log (this->name, GF_LOG_ERROR, "Failing lookup for %s, "
-                        "LOOKUP on a file without gfid is not allowed when "
-                        "some of the children are down", local->loc.path);
+        if (local->op_ret < 0) {
+		local->op_errno = op_errno;
+		local->op_ret = -1;
                 goto unwind;
-        }
+	}
 
-        if ((gfid_miss_count == local->success_count) &&
-            uuid_is_null (local->cont.lookup.gfid_req)) {
-                local->op_ret = -1;
-                local->op_errno = ENODATA;
-                gf_log (this->name, GF_LOG_ERROR, "%s: No gfid present",
-                        local->loc.path);
-                goto unwind;
-        }
+	afr_replies_interpret (frame, this, local->inode);
 
-        if (gfid_miss_count && uuid_is_null (local->cont.lookup.gfid_req))
-                fail_conflict = _gf_true;
-        ret = afr_lookup_done_success_action (frame, this, fail_conflict);
-        if (ret)
-                goto unwind;
-        uuid_copy (local->self_heal.sh_gfid_req, local->cont.lookup.gfid_req);
+	read_subvol = afr_data_subvol_get (local->inode, this, 0, 0);
+	if (read_subvol == -1) {
+		gf_log (this->name, GF_LOG_WARNING, "no read subvols for %s",
+			local->loc.path);
 
-        afr_lookup_perform_self_heal (frame, this, &sh_launched);
-        if (sh_launched) {
-                unwind = 0;
-                goto unwind;
-        }
+		for (i = 0; i < priv->child_count; i++) {
+			if (!local->replies[i].valid ||
+			    local->replies[i].op_ret == -1)
+				continue;
+			read_subvol = i;
+			break;
+		}
+	}
 
- unwind:
-         if (unwind) {
-                 AFR_STACK_UNWIND (lookup, frame, local->op_ret,
-                                   local->op_errno, local->cont.lookup.inode,
-                                   &local->cont.lookup.buf,
-                                   local->cont.lookup.xattr,
-                                   &local->cont.lookup.postparent);
-        }
+unwind:
+	if (read_subvol == -1)
+		read_subvol = 0;
+
+	AFR_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
+			  local->inode, &local->replies[read_subvol].poststat,
+			  local->replies[read_subvol].xdata,
+			  &local->replies[read_subvol].postparent);
 }
 
-/*
- * During a lookup, some errors are more "important" than
- * others in that they must be given higher priority while
- * returning to the user.
- *
- * The hierarchy is ESTALE > ENOENT > others
- *
- */
 
-gf_boolean_t
-afr_error_more_important (int32_t old_errno, int32_t new_errno)
+int
+afr_discover_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		  int op_ret, int op_errno, inode_t *inode, struct iatt *buf,
+		  dict_t *xdata, struct iatt *postparent)
 {
-        gf_boolean_t ret = _gf_true;
+        afr_local_t *   local = NULL;
+        int             call_count      = -1;
+        int             child_index     = -1;
 
-        /* Nothing should ever overwrite ESTALE */
-        if (old_errno == ESTALE)
-                ret = _gf_false;
+	child_index = (long) cookie;
 
-        /* Nothing should overwrite ENOENT, except ESTALE/EIO*/
-        else if ((old_errno == ENOENT) && (new_errno != ESTALE)
-                 && (new_errno != EIO))
-                ret = _gf_false;
+	local = frame->local;
 
-        return ret;
-}
+	local->replies[child_index].valid = 1;
+	local->replies[child_index].op_ret = op_ret;
+	local->replies[child_index].op_errno = op_errno;
+	if (op_ret != -1) {
+		local->replies[child_index].poststat = *buf;
+		local->replies[child_index].postparent = *postparent;
+		if (xdata)
+			local->replies[child_index].xdata = dict_ref (xdata);
+	}
 
-int32_t
-afr_resultant_errno_get (int32_t *children,
-                         int *child_errno, unsigned int child_count)
-{
-        int     i = 0;
-        int32_t op_errno = 0;
-        int     child = 0;
+        if (local->do_discovery && (op_ret == 0))
+                afr_attempt_local_discovery (this, child_index);
 
-        for (i = 0; i < child_count; i++) {
-                if (children) {
-                        child = children[i];
-                        if (child == -1)
-                                break;
-                } else {
-                        child = i;
-                }
-                if (afr_error_more_important (op_errno, child_errno[child]))
-                                op_errno = child_errno[child];
+        call_count = afr_frame_return (frame);
+        if (call_count == 0) {
+               afr_discover_done (frame, this);
         }
-        return op_errno;
+
+	return 0;
 }
 
-static void
-afr_lookup_handle_error (afr_local_t *local, int32_t op_ret,  int32_t op_errno)
+
+int
+afr_discover_do (call_frame_t *frame, xlator_t *this, int err)
 {
-        GF_ASSERT (local);
-        if (op_errno == ENOENT)
-                local->enoent_count++;
+	int ret = 0;
+	int i = 0;
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	int call_count = 0;
 
-        if (afr_error_more_important (local->op_errno, op_errno))
-                local->op_errno = op_errno;
+	local = frame->local;
+	priv = this->private;
 
-        if (local->op_errno == ESTALE) {
-                local->op_ret = -1;
-        }
-}
+	if (err) {
+		local->op_errno = -err;
+		ret = -1;
+		goto out;
+	}
 
-static void
-afr_set_root_inode_on_first_lookup (afr_local_t *local, xlator_t *this,
-                                    inode_t *inode)
-{
-        afr_private_t           *priv = NULL;
-        GF_ASSERT (inode);
+	call_count = local->call_count = AFR_COUNT (local->child_up,
+						    priv->child_count);
 
-        if (!__is_root_gfid (inode->gfid))
-                goto out;
-        if (!afr_is_fresh_lookup (&local->loc, this))
+        ret = afr_lookup_xattr_req_prepare (local, this, local->xattr_req,
+					    &local->loc);
+        if (ret) {
+                local->op_errno = -ret;
+		ret = -1;
                 goto out;
-        priv = this->private;
-        if ((priv->first_lookup)) {
-                gf_log (this->name, GF_LOG_INFO, "added root inode");
-                priv->root_inode = inode_ref (inode);
-                priv->first_lookup = 0;
         }
-out:
-        return;
-}
-
-static void
-afr_lookup_cache_args (afr_local_t *local, int child_index, dict_t *xattr,
-                       struct iatt *buf, struct iatt *postparent)
-{
-        GF_ASSERT (child_index >= 0);
-        local->cont.lookup.xattrs[child_index] = dict_ref (xattr);
-        local->cont.lookup.postparents[child_index] = *postparent;
-        local->cont.lookup.bufs[child_index] = *buf;
-}
 
-static void
-afr_lookup_handle_first_success (afr_local_t *local, xlator_t *this,
-                                 inode_t *inode, struct iatt *buf)
-{
-        local->cont.lookup.inode      = inode_ref (inode);
-        local->cont.lookup.buf        = *buf;
-        afr_set_root_inode_on_first_lookup (local, this, inode);
-}
-
-static void
-afr_lookup_handle_success (afr_local_t *local, xlator_t *this, int32_t child_index,
-                           int32_t op_ret, int32_t op_errno, inode_t *inode,
-                           struct iatt *buf, dict_t *xattr,
-                           struct iatt *postparent)
-{
-        if (local->success_count == 0) {
-                if (local->op_errno != ESTALE) {
-                        local->op_ret = op_ret;
-                        local->op_errno = 0;
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->child_up[i]) {
+                        STACK_WIND_COOKIE (frame, afr_discover_cbk,
+                                           (void *) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->lookup,
+                                           &local->loc, local->xattr_req);
+                        if (!--call_count)
+                                break;
                 }
-                afr_lookup_handle_first_success (local, this, inode, buf);
         }
-        afr_lookup_update_lk_counts (local, this,
-                                     child_index, xattr);
 
-        afr_lookup_cache_args (local, child_index, xattr,
-                               buf, postparent);
-        local->cont.lookup.success_children[local->success_count] = child_index;
-        local->success_count++;
+	return 0;
+out:
+	AFR_STACK_UNWIND (lookup, frame, -1, local->op_errno, 0, 0, 0, 0);
+	return 0;
 }
 
+
 int
-afr_lookup_cbk (call_frame_t *frame, void *cookie,
-                xlator_t *this,  int32_t op_ret,  int32_t op_errno,
-                inode_t *inode,   struct iatt *buf, dict_t *xattr,
-                struct iatt *postparent)
+afr_discover (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr_req)
 {
-        afr_local_t *   local = NULL;
-        int             call_count      = -1;
-        int             child_index     = -1;
+	int op_errno = ENOMEM;
+	afr_private_t *priv = NULL;
+	afr_local_t *local = NULL;
+	int event = 0;
 
-         child_index = (long) cookie;
+	priv = this->private;
 
-        LOCK (&frame->lock);
-        {
-                local = frame->local;
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
 
-                if (op_ret == -1) {
-                        afr_lookup_handle_error (local, op_ret, op_errno);
-                        goto unlock;
-                }
-                afr_lookup_handle_success (local, this, child_index, op_ret,
-                                           op_errno, inode, buf, xattr,
-                                           postparent);
+        if (!local->call_count) {
+                op_errno = ENOTCONN;
+                goto out;
+        }
 
-         }
-unlock:
-        UNLOCK (&frame->lock);
+	if (__is_root_gfid (loc->inode->gfid)) {
+		if (!this->itable)
+			this->itable = loc->inode->table;
+		if (!priv->root_inode)
+			priv->root_inode = inode_ref (loc->inode);
 
-        call_count = afr_frame_return (frame);
-        if (call_count == 0) {
-               afr_lookup_done (frame, this);
-        }
+		if (priv->choose_local && !priv->did_discovery) {
+			/* Logic to detect which subvolumes of AFR are
+			   local, in order to prefer them for reads
+			*/
+			local->do_discovery = _gf_true;
+                        priv->did_discovery = _gf_true;
+                }
+	}
 
-         return 0;
-}
+        local->op = GF_FOP_LOOKUP;
 
-int
-afr_lookup_cont_init (afr_local_t *local, unsigned int child_count)
-{
-        int               ret            = -ENOMEM;
-        struct iatt       *iatts         = NULL;
-        int32_t           *success_children = NULL;
-        int32_t           *sources       = NULL;
-
-        GF_ASSERT (local);
-        local->cont.lookup.xattrs = GF_CALLOC (child_count,
-                                               sizeof (*local->cont.lookup.xattr),
-                                               gf_afr_mt_dict_t);
-        if (NULL == local->cont.lookup.xattrs)
-                goto out;
+        loc_copy (&local->loc, loc);
 
-        iatts = GF_CALLOC (child_count, sizeof (*iatts), gf_afr_mt_iatt);
-        if (NULL == iatts)
-                goto out;
-        local->cont.lookup.postparents = iatts;
+	local->inode = inode_ref (loc->inode);
 
-        iatts = GF_CALLOC (child_count, sizeof (*iatts), gf_afr_mt_iatt);
-        if (NULL == iatts)
-                goto out;
-        local->cont.lookup.bufs = iatts;
+	if (xattr_req)
+		/* If xattr_req was null, afr_lookup_xattr_req_prepare() will
+		   allocate one for us */
+		local->xattr_req = dict_ref (xattr_req);
 
-        success_children = afr_children_create (child_count);
-        if (NULL == success_children)
-                goto out;
-        local->cont.lookup.success_children = success_children;
+	if (uuid_is_null (loc->inode->gfid)) {
+		afr_discover_do (frame, this, 0);
+		return 0;
+	}
 
-        local->fresh_children = afr_children_create (child_count);
-        if (NULL == local->fresh_children)
-                goto out;
+	afr_read_subvol_get (loc->inode, this, NULL, &event,
+			     AFR_DATA_TRANSACTION);
 
-        sources = GF_CALLOC (sizeof (*sources), child_count, gf_afr_mt_int32_t);
-        if (NULL == sources)
-                goto out;
-        local->cont.lookup.sources = sources;
+	if (event != local->event_generation)
+		afr_inode_refresh (frame, this, loc->inode, afr_discover_do);
+	else
+		afr_discover_do (frame, this, 0);
 
-        ret = 0;
+	return 0;
 out:
-        return ret;
+	AFR_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+	return 0;
 }
 
+
 int
-afr_lookup (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, dict_t *xattr_req)
+afr_lookup_do (call_frame_t *frame, xlator_t *this, int err)
 {
-        afr_private_t    *priv           = NULL;
-        afr_local_t      *local          = NULL;
-        void              *gfid_req      = NULL;
-        int               ret            = -1;
-        int               i              = 0;
-        int               call_count     = 0;
-        uint64_t          ctx            = 0;
-        int32_t           op_errno       = 0;
-
-        priv = this->private;
-
-        AFR_LOCAL_ALLOC_OR_GOTO (local, out);
+	int ret = 0;
+	int i = 0;
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	int call_count = 0;
 
-        local->op_ret = -1;
-
-        frame->local = local;
-        local->fop = GF_FOP_LOOKUP;
-
-        loc_copy (&local->loc, loc);
-        ret = loc_path (&local->loc, NULL);
-        if (ret < 0) {
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        if (!strcmp (local->loc.path, "/" GF_REPLICATE_TRASH_DIR)) {
-                op_errno = ENOENT;
-                goto out;
-        }
-
-        ret = inode_ctx_get (local->loc.inode, this, &ctx);
-        if (ret == 0) {
-                /* lookup is a revalidate */
-
-                local->read_child_index = afr_inode_get_read_ctx (this,
-                                                               local->loc.inode,
-                                                               NULL);
-        } else {
-                LOCK (&priv->read_child_lock);
-                {
-                        local->read_child_index = (++priv->read_child_rr)
-                                % (priv->child_count);
-                }
-                UNLOCK (&priv->read_child_lock);
-                local->cont.lookup.fresh_lookup = _gf_true;
-        }
-
-        local->child_up = memdup (priv->child_up,
-                                  sizeof (*local->child_up) * priv->child_count);
-        if (NULL == local->child_up) {
-                op_errno = ENOMEM;
-                goto out;
-        }
+	local = frame->local;
+	priv = this->private;
 
-        ret = afr_lookup_cont_init (local, priv->child_count);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
-        }
-
-        local->call_count = afr_up_children_count (local->child_up,
-                                                   priv->child_count);
-        call_count = local->call_count;
-        if (local->call_count == 0) {
-                ret      = -1;
-                op_errno = ENOTCONN;
-                goto out;
-        }
+	if (err < 0) {
+		local->op_errno = -err;
+		ret = -1;
+		goto out;
+	}
 
-        /* By default assume ENOTCONN. On success it will be set to 0. */
-        local->op_errno = ENOTCONN;
+	call_count = local->call_count = AFR_COUNT (local->child_up,
+						    priv->child_count);
 
-        local->call_count = afr_up_children_count (local->child_up,
-                                                   priv->child_count);
-        ret = afr_lookup_xattr_req_prepare (local, this, xattr_req, &local->loc,
-                                            &gfid_req);
+        ret = afr_lookup_xattr_req_prepare (local, this, local->xattr_req,
+					    &local->loc);
         if (ret) {
                 local->op_errno = -ret;
+		ret = -1;
                 goto out;
         }
-        afr_lookup_save_gfid (local->cont.lookup.gfid_req, gfid_req,
-                              &local->loc);
-        local->fop = GF_FOP_LOOKUP;
+
         for (i = 0; i < priv->child_count; i++) {
                 if (local->child_up[i]) {
                         STACK_WIND_COOKIE (frame, afr_lookup_cbk,
@@ -2246,12 +1778,98 @@ afr_lookup (call_frame_t *frame, xlator_t *this,
                                 break;
                 }
         }
+	return 0;
+out:
+	AFR_STACK_UNWIND (lookup, frame, -1, local->op_errno, 0, 0, 0, 0);
+	return 0;
+}
 
-        ret = 0;
+/*
+ * afr_lookup()
+ *
+ * The goal here is to figure out what the element getting looked up is.
+ * i.e what is the GFID, inode type and a conservative estimate of the
+ * inode attributes are.
+ *
+ * As we lookup, operations may be underway on the entry name and the
+ * inode. In lookup() we are primarily concerned only with the entry
+ * operations. If the entry is getting unlinked or renamed, we detect
+ * what operation is underway by querying for on-going transactions and
+ * pending self-healing on the entry through xdata.
+ *
+ * If the entry is a file/dir, it may need self-heal and/or in a
+ * split-brain condition. Lookup is not the place to worry about these
+ * conditions. Outcast marking will naturally handle them in the read
+ * paths.
+ *
+ * Here is a brief goal of what we are trying to achieve:
+ *
+ * - LOOKUP on all subvolumes concurrently, querying on-going transaction
+ *   and pending self-heal info from the servers.
+ *
+ * - If all servers reply the same inode type and GFID, the overall call
+ *   MUST be a success.
+ *
+ * - If inode types or GFIDs mismatch, and there IS either an on-going
+ *   transaction or pending self-heal, inspect what the nature of the
+ *   transaction or pending heal is, and select the appropriate subvolume's
+ *   reply as the winner.
+ *
+ * - If inode types or GFIDs mismatch, and there are no on-going transactions
+ *   or pending self-heal on the entry name on any of the servers, fail the
+ *   lookup with EIO. Something has gone wrong beyond reasonable action.
+ */
+
+int
+afr_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr_req)
+{
+        afr_local_t   *local = NULL;
+        int32_t        op_errno = 0;
+	int            event = 0;
+
+	if (!loc->parent) {
+		afr_discover (frame, this, loc, xattr_req);
+		return 0;
+	}
+
+	if (__is_root_gfid (loc->parent->gfid)) {
+		if (!strcmp (loc->name, GF_REPLICATE_TRASH_DIR)) {
+			op_errno = EPERM;
+			goto out;
+		}
+	}
+
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
+
+        if (!local->call_count) {
+                op_errno = ENOTCONN;
+                goto out;
+        }
+
+        local->op = GF_FOP_LOOKUP;
+
+        loc_copy (&local->loc, loc);
+
+	local->inode = inode_ref (loc->inode);
+
+	if (xattr_req)
+		/* If xattr_req was null, afr_lookup_xattr_req_prepare() will
+		   allocate one for us */
+		local->xattr_req = dict_ref (xattr_req);
+
+	afr_read_subvol_get (loc->parent, this, NULL, &event,
+			     AFR_DATA_TRANSACTION);
+
+	if (event != local->event_generation)
+		afr_inode_refresh (frame, this, loc->parent, afr_lookup_do);
+	else
+		afr_lookup_do (frame, this, 0);
+
+	return 0;
 out:
-        if (ret)
-                AFR_STACK_UNWIND (lookup, frame, -1, op_errno,
-                                  NULL, NULL, NULL, NULL);
+	AFR_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
 
         return 0;
 }
@@ -2259,6 +1877,46 @@ out:
 
 /* {{{ open */
 
+afr_fd_ctx_t *
+__afr_fd_ctx_get (fd_t *fd, xlator_t *this)
+{
+        uint64_t       ctx = 0;
+        int            ret = 0;
+        afr_fd_ctx_t  *fd_ctx = NULL;
+
+        ret = __fd_ctx_get (fd, this, &ctx);
+
+        if (ret < 0) {
+                ret = __afr_fd_ctx_set (this, fd);
+                if (ret < 0)
+                        goto out;
+
+                ret = __fd_ctx_get (fd, this, &ctx);
+                if (ret < 0)
+                        goto out;
+        }
+
+        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
+out:
+        return fd_ctx;
+}
+
+
+afr_fd_ctx_t *
+afr_fd_ctx_get (fd_t *fd, xlator_t *this)
+{
+        afr_fd_ctx_t  *fd_ctx = NULL;
+
+        LOCK(&fd->lock);
+        {
+                fd_ctx = __afr_fd_ctx_get (fd, this);
+        }
+        UNLOCK(&fd->lock);
+
+        return fd_ctx;
+}
+
+
 int
 __afr_fd_ctx_set (xlator_t *this, fd_t *fd)
 {
@@ -2266,6 +1924,7 @@ __afr_fd_ctx_set (xlator_t *this, fd_t *fd)
         int             ret    = -1;
         uint64_t        ctx    = 0;
         afr_fd_ctx_t *  fd_ctx = NULL;
+	int             i = 0;
 
         VALIDATE_OR_GOTO (this->private, out);
         VALIDATE_OR_GOTO (fd, out);
@@ -2284,21 +1943,15 @@ __afr_fd_ctx_set (xlator_t *this, fd_t *fd)
                 goto out;
         }
 
-        fd_ctx->pre_op_done = GF_CALLOC (sizeof (*fd_ctx->pre_op_done),
-                                         priv->child_count,
-                                         gf_afr_mt_char);
-        if (!fd_ctx->pre_op_done) {
-                ret = -ENOMEM;
-                goto out;
-        }
-
-        fd_ctx->pre_op_piggyback = GF_CALLOC (sizeof (*fd_ctx->pre_op_piggyback),
-                                              priv->child_count,
-                                              gf_afr_mt_char);
-        if (!fd_ctx->pre_op_piggyback) {
-                ret = -ENOMEM;
-                goto out;
-        }
+	for (i = 0; i < AFR_NUM_CHANGE_LOGS; i++) {
+		fd_ctx->pre_op_done[i] = GF_CALLOC (sizeof (*fd_ctx->pre_op_done[i]),
+						    priv->child_count,
+						    gf_afr_mt_int32_t);
+		if (!fd_ctx->pre_op_done[i]) {
+			ret = -ENOMEM;
+			goto out;
+		}
+	}
 
         fd_ctx->opened_on = GF_CALLOC (sizeof (*fd_ctx->opened_on),
                                        priv->child_count,
@@ -2308,6 +1961,13 @@ __afr_fd_ctx_set (xlator_t *this, fd_t *fd)
                 goto out;
         }
 
+	for (i = 0; i < priv->child_count; i++) {
+		if (fd_is_anonymous (fd))
+			fd_ctx->opened_on[i] = AFR_FD_OPENED;
+		else
+			fd_ctx->opened_on[i] = AFR_FD_NOT_OPENED;
+	}
+
         fd_ctx->lock_piggyback = GF_CALLOC (sizeof (*fd_ctx->lock_piggyback),
                                             priv->child_count,
                                             gf_afr_mt_char);
@@ -2324,19 +1984,9 @@ __afr_fd_ctx_set (xlator_t *this, fd_t *fd)
                 goto out;
         }
 
-        fd_ctx->up_count   = priv->up_count;
-        fd_ctx->down_count = priv->down_count;
-
-        fd_ctx->locked_on = GF_CALLOC (sizeof (*fd_ctx->locked_on),
-                                       priv->child_count,
-                                       gf_afr_mt_char);
-        if (!fd_ctx->locked_on) {
-                ret = -ENOMEM;
-                goto out;
-        }
+	pthread_mutex_init (&fd_ctx->delay_lock, NULL);
 
-        INIT_LIST_HEAD (&fd_ctx->paused_calls);
-        INIT_LIST_HEAD (&fd_ctx->entries);
+        INIT_LIST_HEAD (&fd_ctx->eager_locked);
 
         ret = __fd_ctx_set (fd, this, (uint64_t)(long) fd_ctx);
         if (ret)
@@ -2364,186 +2014,90 @@ afr_fd_ctx_set (xlator_t *this, fd_t *fd)
 /* {{{ flush */
 
 int
-afr_flush_unwind (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
-
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame)
-                        main_frame = local->transaction.main_frame;
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
-
-        if (main_frame) {
-                AFR_STACK_UNWIND (flush, main_frame,
-                                  local->op_ret, local->op_errno,
-                                  NULL);
-        }
-
-        return 0;
-}
-
-
-int
-afr_flush_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+afr_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
-        int call_count  = -1;
-        int child_index = (long) cookie;
-        int need_unwind = 0;
+        afr_local_t *local = NULL;
+        int call_count = -1;
 
         local = frame->local;
-        priv  = this->private;
 
         LOCK (&frame->lock);
         {
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
-
                 if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                        }
-                        local->success_count++;
-
-                        if (local->success_count == priv->wait_count) {
-                                need_unwind = 1;
-                        }
-                }
-
-                local->op_errno = op_errno;
+			local->op_ret = op_ret;
+			if (!local->xdata_rsp && xdata)
+				local->xdata_rsp = dict_ref (xdata);
+		} else {
+			local->op_errno = op_errno;
+		}
         }
         UNLOCK (&frame->lock);
 
-        if (need_unwind)
-                afr_flush_unwind (frame, this);
+	call_count = afr_frame_return (frame);
 
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-        }
+	if (call_count == 0)
+		AFR_STACK_UNWIND (flush, frame, local->op_ret,
+				  local->op_errno, local->xdata_rsp);
 
         return 0;
 }
 
-
-int
-afr_flush_wind (call_frame_t *frame, xlator_t *this)
+static int
+afr_flush_wrapper (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        int i = 0;
-        int call_count = -1;
+        int           i      = 0;
+        afr_local_t   *local = NULL;
+        afr_private_t *priv  = NULL;
+        int call_count       = -1;
 
-        local = frame->local;
         priv = this->private;
-
-        call_count = afr_up_children_count (local->child_up, priv->child_count);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
-
-        local->call_count = call_count;
+        local = frame->local;
+        call_count = local->call_count;
 
         for (i = 0; i < priv->child_count; i++) {
                 if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, afr_flush_wind_cbk,
+                        STACK_WIND_COOKIE (frame, afr_flush_cbk,
                                            (void *) (long) i,
                                            priv->children[i],
                                            priv->children[i]->fops->flush,
-                                           local->fd, NULL);
-
+                                           local->fd, xdata);
                         if (!--call_count)
                                 break;
+
                 }
         }
 
         return 0;
 }
 
-
-int
-afr_flush_done (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local = NULL;
-
-        local = frame->local;
-
-        local->transaction.unwind (frame, this);
-
-        AFR_STACK_DESTROY (frame);
-
-        return 0;
-}
-
-
 int
 afr_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        afr_private_t *priv  = NULL;
         afr_local_t   *local = NULL;
-        call_frame_t  *transaction_frame = NULL;
-        int            ret        = -1;
-        int            op_errno   = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
-
-        priv = this->private;
+        call_stub_t   *stub = NULL;
+        int            op_errno   = ENOMEM;
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
-        local = transaction_frame->local;
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
-
-        local->op = GF_FOP_FLUSH;
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
 
-        local->transaction.fop    = afr_flush_wind;
-        local->transaction.done   = afr_flush_done;
-        local->transaction.unwind = afr_flush_unwind;
-
-        local->fd                 = fd_ref (fd);
+	if (!local->call_count) {
+		op_errno = ENOTCONN;
+		goto out;
+	}
 
-        local->transaction.main_frame = frame;
-        local->transaction.start  = 0;
-        local->transaction.len    = 0;
+	local->fd = fd_ref(fd);
 
-        ret = afr_open_fd_fix (transaction_frame, this, _gf_false);
-        if (ret) {
-                op_errno = -ret;
+        stub = fop_flush_stub (frame, afr_flush_wrapper, fd, xdata);
+        if (!stub)
                 goto out;
-        }
-        afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
 
+        afr_delayed_changelog_wake_resume (this, fd, stub);
 
-        ret = 0;
+	return 0;
 out:
-        if (ret < 0) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-
-                AFR_STACK_UNWIND (flush, frame, -1, op_errno, NULL);
-        }
-
+	AFR_STACK_UNWIND (flush, frame, -1, op_errno, NULL);
         return 0;
 }
 
@@ -2556,8 +2110,7 @@ afr_cleanup_fd_ctx (xlator_t *this, fd_t *fd)
         uint64_t        ctx = 0;
         afr_fd_ctx_t    *fd_ctx = NULL;
         int             ret = 0;
-        afr_fd_paused_call_t *paused_call = NULL;
-        afr_fd_paused_call_t *tmp = NULL;
+	int             i = 0;
 
         ret = fd_ctx_get (fd, this, &ctx);
         if (ret < 0)
@@ -2566,28 +2119,22 @@ afr_cleanup_fd_ctx (xlator_t *this, fd_t *fd)
         fd_ctx = (afr_fd_ctx_t *)(long) ctx;
 
         if (fd_ctx) {
-                if (fd_ctx->pre_op_done)
-                        GF_FREE (fd_ctx->pre_op_done);
+                //no need to take any locks
+                if (!list_empty (&fd_ctx->eager_locked))
+                        gf_log (this->name, GF_LOG_WARNING, "%s: Stale "
+                                "Eager-lock stubs found",
+                                uuid_utoa (fd->inode->gfid));
 
-                if (fd_ctx->opened_on)
-                        GF_FREE (fd_ctx->opened_on);
+		for (i = 0; i < AFR_NUM_CHANGE_LOGS; i++)
+			GF_FREE (fd_ctx->pre_op_done[i]);
 
-                if (fd_ctx->locked_on)
-                        GF_FREE (fd_ctx->locked_on);
+                GF_FREE (fd_ctx->opened_on);
 
-                if (fd_ctx->pre_op_piggyback)
-                        GF_FREE (fd_ctx->pre_op_piggyback);
-                list_for_each_entry_safe (paused_call, tmp, &fd_ctx->paused_calls,
-                                          call_list) {
-                        list_del_init (&paused_call->call_list);
-                        GF_FREE (paused_call);
-                }
+                GF_FREE (fd_ctx->lock_piggyback);
 
-                if (fd_ctx->lock_piggyback)
-                        GF_FREE (fd_ctx->lock_piggyback);
+                GF_FREE (fd_ctx->lock_acquired);
 
-                if (fd_ctx->lock_acquired)
-                        GF_FREE (fd_ctx->lock_acquired);
+		pthread_mutex_destroy (&fd_ctx->delay_lock);
 
                 GF_FREE (fd_ctx);
         }
@@ -2600,24 +2147,8 @@ out:
 int
 afr_release (xlator_t *this, fd_t *fd)
 {
-        afr_locked_fd_t *locked_fd = NULL;
-        afr_locked_fd_t *tmp       = NULL;
-        afr_private_t   *priv      = NULL;
-
-        priv = this->private;
-
         afr_cleanup_fd_ctx (this, fd);
 
-        list_for_each_entry_safe (locked_fd, tmp, &priv->saved_fds,
-                                  list) {
-
-                if (locked_fd->fd == fd) {
-                        list_del_init (&locked_fd->list);
-                        GF_FREE (locked_fd);
-                }
-
-        }
-
         return 0;
 }
 
@@ -2625,6 +2156,16 @@ afr_release (xlator_t *this, fd_t *fd)
 /* {{{ fsync */
 
 int
+afr_fsync_unwind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xdata)
+{
+        AFR_STACK_UNWIND (fsync, frame, op_ret, op_errno, prebuf, postbuf,
+                          xdata);
+        return 0;
+}
+
+int
 afr_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
                struct iatt *postbuf, dict_t *xdata)
@@ -2632,45 +2173,70 @@ afr_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         afr_local_t *local = NULL;
         int call_count = -1;
         int child_index = (long) cookie;
-        int read_child  = 0;
+	int read_subvol = 0;
+	call_stub_t *stub = NULL;
 
         local = frame->local;
 
-        read_child = afr_inode_get_read_ctx (this, local->fd->inode, NULL);
+	read_subvol = afr_data_subvol_get (local->inode, this, 0, 0);
 
         LOCK (&frame->lock);
         {
-                if (child_index == read_child) {
-                        local->read_child_returned = _gf_true;
-                }
-
                 if (op_ret == 0) {
-                        local->op_ret = 0;
+                        if (local->op_ret == -1) {
+				local->op_ret = 0;
 
-                        if (local->success_count == 0) {
-                                local->cont.fsync.prebuf  = *prebuf;
-                                local->cont.fsync.postbuf = *postbuf;
-                        }
+                                local->cont.inode_wfop.prebuf  = *prebuf;
+                                local->cont.inode_wfop.postbuf = *postbuf;
 
-                        if (child_index == read_child) {
-                                local->cont.fsync.prebuf  = *prebuf;
-                                local->cont.fsync.postbuf = *postbuf;
+				if (xdata)
+					local->xdata_rsp = dict_ref (xdata);
                         }
 
-                        local->success_count++;
-                }
-
-                local->op_errno = op_errno;
+                        if (child_index == read_subvol) {
+                                local->cont.inode_wfop.prebuf  = *prebuf;
+                                local->cont.inode_wfop.postbuf = *postbuf;
+				if (xdata) {
+					if (local->xdata_rsp)
+						dict_unref (local->xdata_rsp);
+					local->xdata_rsp = dict_ref (xdata);
+				}
+                        }
+                } else {
+			local->op_errno = op_errno;
+		}
         }
         UNLOCK (&frame->lock);
 
         call_count = afr_frame_return (frame);
 
         if (call_count == 0) {
-                AFR_STACK_UNWIND (fsync, frame, local->op_ret, local->op_errno,
-                                  &local->cont.fsync.prebuf,
-                                  &local->cont.fsync.postbuf,
-                                  NULL);
+		/* Make a stub out of the frame, and register it
+		   with the waking up post-op. When the call-stub resumes,
+		   we are guaranteed that there was no post-op pending
+		   (i.e changelogs were unset in the server). This is an
+		   essential "guarantee", that fsync() returns only after
+		   completely finishing EVERYTHING, including the delayed
+		   post-op. This guarantee is expected by FUSE graph switching
+		   for example.
+		*/
+		stub = fop_fsync_cbk_stub (frame, afr_fsync_unwind_cbk,
+                                           local->op_ret, local->op_errno,
+                                           &local->cont.inode_wfop.prebuf,
+                                           &local->cont.inode_wfop.postbuf,
+                                           local->xdata_rsp);
+		if (!stub) {
+			AFR_STACK_UNWIND (fsync, frame, -1, ENOMEM, 0, 0, 0);
+			return 0;
+		}
+
+		/* If no new unstable writes happened between the
+		   time we cleared the unstable write witness flag in afr_fsync
+		   and now, calling afr_delayed_changelog_wake_up() should
+		   wake up and skip over the fsync phase and go straight to
+		   afr_changelog_post_op_now()
+		*/
+		afr_delayed_changelog_wake_resume (this, local->fd, stub);
         }
 
         return 0;
@@ -2678,32 +2244,34 @@ afr_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 
 int
-afr_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd,
-           int32_t datasync, dict_t *xdata)
+afr_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+	   dict_t *xdata)
 {
-        afr_private_t *priv = NULL;
+	afr_private_t *priv = NULL;
         afr_local_t *local = NULL;
-        int ret = -1;
         int i = 0;
         int32_t call_count = 0;
-        int32_t op_errno = 0;
+        int32_t op_errno = ENOMEM;
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+	priv = this->private;
 
-        priv = this->private;
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
 
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
+        call_count = local->call_count;
+	if (!call_count) {
+		op_errno = ENOTCONN;
+		goto out;
+	}
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+        local->fd = fd_ref (fd);
 
-        call_count = local->call_count;
+	if (afr_fd_has_witnessed_unstable_write (this, fd)) {
+		/* don't care. we only wanted to CLEAR the bit */
+	}
 
-        local->fd             = fd_ref (fd);
+	local->inode = inode_ref (fd->inode);
 
         for (i = 0; i < priv->child_count; i++) {
                 if (local->child_up[i]) {
@@ -2717,10 +2285,10 @@ afr_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd,
                 }
         }
 
-        ret = 0;
+	return 0;
 out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (fsync, frame, -1, op_errno, NULL, NULL, NULL);
+	AFR_STACK_UNWIND (fsync, frame, -1, op_errno, NULL, NULL, NULL);
+
         return 0;
 }
 
@@ -2728,10 +2296,9 @@ out:
 
 /* {{{ fsync */
 
-int32_t
-afr_fsyncdir_cbk (call_frame_t *frame, void *cookie,
-                  xlator_t *this, int32_t op_ret, int32_t op_errno,
-                  dict_t *xdata)
+int
+afr_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		  int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         afr_local_t *local = NULL;
         int call_count = -1;
@@ -2740,10 +2307,13 @@ afr_fsyncdir_cbk (call_frame_t *frame, void *cookie,
 
         LOCK (&frame->lock);
         {
-                if (op_ret == 0)
+                if (op_ret == 0) {
                         local->op_ret = 0;
-
-                local->op_errno = op_errno;
+			if (!local->xdata_rsp && xdata)
+				local->xdata_rsp = dict_ref (xdata);
+		} else {
+			local->op_errno = op_errno;
+		}
         }
         UNLOCK (&frame->lock);
 
@@ -2751,37 +2321,33 @@ afr_fsyncdir_cbk (call_frame_t *frame, void *cookie,
 
         if (call_count == 0)
                 AFR_STACK_UNWIND (fsyncdir, frame, local->op_ret,
-                                  local->op_errno, xdata);
+				  local->op_errno, local->xdata_rsp);
 
         return 0;
 }
 
 
-int32_t
-afr_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
-              int32_t datasync, dict_t *xdata)
+int
+afr_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+	      dict_t *xdata)
 {
-        afr_private_t *priv = NULL;
+	afr_private_t *priv = NULL;
         afr_local_t *local = NULL;
-        int ret = -1;
         int i = 0;
         int32_t call_count = 0;
-        int32_t op_errno = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+        int32_t op_errno = ENOMEM;
 
-        priv = this->private;
-
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
+	priv = this->private;
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+        local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
 
         call_count = local->call_count;
+	if (!call_count) {
+		op_errno = ENOTCONN;
+		goto out;
+	}
 
         for (i = 0; i < priv->child_count; i++) {
                 if (local->child_up[i]) {
@@ -2794,10 +2360,10 @@ afr_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
                 }
         }
 
-        ret = 0;
+	return 0;
 out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (fsyncdir, frame, -1, op_errno, NULL);
+	AFR_STACK_UNWIND (fsyncdir, frame, -1, op_errno, NULL);
+
         return 0;
 }
 
@@ -2820,6 +2386,10 @@ afr_xattrop_cbk (call_frame_t *frame, void *cookie,
                 if (op_ret == 0) {
                         if (!local->cont.xattrop.xattr)
                                 local->cont.xattrop.xattr = dict_ref (xattr);
+
+			if (!local->xdata_rsp && xdata)
+				local->xdata_rsp = dict_ref (xdata);
+
                         local->op_ret = 0;
                 }
 
@@ -2831,7 +2401,7 @@ afr_xattrop_cbk (call_frame_t *frame, void *cookie,
 
         if (call_count == 0)
                 AFR_STACK_UNWIND (xattrop, frame, local->op_ret, local->op_errno,
-                local->cont.xattrop.xattr, xdata);
+                local->cont.xattrop.xattr, local->xdata_rsp);
 
         return 0;
 }
@@ -2843,25 +2413,21 @@ afr_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
 {
         afr_private_t *priv = NULL;
         afr_local_t *local  = NULL;
-        int ret = -1;
         int i = 0;
         int32_t call_count = 0;
-        int32_t op_errno = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+        int32_t op_errno = ENOMEM;
 
         priv = this->private;
 
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
 
         call_count = local->call_count;
+	if (!call_count) {
+		op_errno = ENOTCONN;
+		goto out;
+	}
 
         for (i = 0; i < priv->child_count; i++) {
                 if (local->child_up[i]) {
@@ -2874,10 +2440,10 @@ afr_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
                 }
         }
 
-        ret = 0;
+	return 0;
 out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (xattrop, frame, -1, op_errno, NULL, NULL);
+	AFR_STACK_UNWIND (xattrop, frame, -1, op_errno, NULL, NULL);
+
         return 0;
 }
 
@@ -2902,6 +2468,8 @@ afr_fxattrop_cbk (call_frame_t *frame, void *cookie,
                         if (!local->cont.fxattrop.xattr)
                                 local->cont.fxattrop.xattr = dict_ref (xattr);
 
+			if (!local->xdata_rsp && xdata)
+				local->xdata_rsp = dict_ref (xdata);
                         local->op_ret = 0;
                 }
 
@@ -2913,7 +2481,7 @@ afr_fxattrop_cbk (call_frame_t *frame, void *cookie,
 
         if (call_count == 0)
                 AFR_STACK_UNWIND (fxattrop, frame, local->op_ret, local->op_errno,
-                                  local->cont.fxattrop.xattr, xdata);
+                                  local->cont.fxattrop.xattr, local->xdata_rsp);
 
         return 0;
 }
@@ -2925,25 +2493,21 @@ afr_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
 {
         afr_private_t *priv = NULL;
         afr_local_t *local  = NULL;
-        int ret = -1;
         int i = 0;
         int32_t call_count = 0;
         int32_t op_errno = 0;
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
-
         priv = this->private;
 
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
                 goto out;
 
         call_count = local->call_count;
+	if (!call_count) {
+		op_errno = ENOTCONN;
+		goto out;
+	}
 
         for (i = 0; i < priv->child_count; i++) {
                 if (local->child_up[i]) {
@@ -2956,10 +2520,10 @@ afr_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
                 }
         }
 
-        ret = 0;
+	return 0;
 out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (fxattrop, frame, -1, op_errno, NULL, NULL);
+	AFR_STACK_UNWIND (fxattrop, frame, -1, op_errno, NULL, NULL);
+
         return 0;
 }
 
@@ -2967,8 +2531,8 @@ out:
 
 
 int32_t
-afr_inodelk_cbk (call_frame_t *frame, void *cookie,
-                 xlator_t *this, int32_t op_ret, int32_t op_errno, dict_t *xdata)
+afr_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		 int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
 {
         afr_local_t *local = NULL;
@@ -3002,25 +2566,21 @@ afr_inodelk (call_frame_t *frame, xlator_t *this,
 {
         afr_private_t *priv = NULL;
         afr_local_t *local  = NULL;
-        int ret = -1;
         int i = 0;
         int32_t call_count = 0;
-        int32_t op_errno = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+        int32_t op_errno = ENOMEM;
 
         priv = this->private;
 
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
+        local = AFR_FRAME_INIT (frame, op_errno);
+        if (!local)
                 goto out;
 
         call_count = local->call_count;
+	if (!call_count) {
+		op_errno = ENOMEM;
+		goto out;
+	}
 
         for (i = 0; i < priv->child_count; i++) {
                 if (local->child_up[i]) {
@@ -3034,18 +2594,17 @@ afr_inodelk (call_frame_t *frame, xlator_t *this,
                 }
         }
 
-        ret = 0;
+	return 0;
 out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (inodelk, frame, -1, op_errno, NULL);
+	AFR_STACK_UNWIND (inodelk, frame, -1, op_errno, NULL);
+
         return 0;
 }
 
 
 int32_t
-afr_finodelk_cbk (call_frame_t *frame, void *cookie,
-                  xlator_t *this, int32_t op_ret, int32_t op_errno,
-                  dict_t *xdata)
+afr_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		  int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
 {
         afr_local_t *local = NULL;
@@ -3073,31 +2632,26 @@ afr_finodelk_cbk (call_frame_t *frame, void *cookie,
 
 
 int32_t
-afr_finodelk (call_frame_t *frame, xlator_t *this,
-              const char *volume, fd_t *fd, int32_t cmd, struct gf_flock *flock,
-              dict_t *xdata)
+afr_finodelk (call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+	      int32_t cmd, struct gf_flock *flock, dict_t *xdata)
 {
         afr_private_t *priv = NULL;
         afr_local_t *local  = NULL;
-        int ret = -1;
         int i = 0;
         int32_t call_count = 0;
-        int32_t op_errno = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+        int32_t op_errno = ENOMEM;
 
         priv = this->private;
 
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
 
         call_count = local->call_count;
+	if (!call_count) {
+		op_errno = ENOTCONN;
+		goto out;
+	}
 
         for (i = 0; i < priv->child_count; i++) {
                 if (local->child_up[i]) {
@@ -3111,10 +2665,10 @@ afr_finodelk (call_frame_t *frame, xlator_t *this,
                 }
         }
 
-        ret = 0;
+	return 0;
 out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (finodelk, frame, -1, op_errno, NULL);
+	AFR_STACK_UNWIND (finodelk, frame, -1, op_errno, NULL);
+
         return 0;
 }
 
@@ -3147,33 +2701,28 @@ afr_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 }
 
 
-int32_t
-afr_entrylk (call_frame_t *frame, xlator_t *this,
-             const char *volume, loc_t *loc,
-             const char *basename, entrylk_cmd cmd, entrylk_type type,
-             dict_t *xdata)
+int
+afr_entrylk (call_frame_t *frame, xlator_t *this, const char *volume,
+	     loc_t *loc, const char *basename, entrylk_cmd cmd,
+	     entrylk_type type, dict_t *xdata)
 {
         afr_private_t *priv = NULL;
         afr_local_t *local  = NULL;
-        int ret = -1;
         int i = 0;
         int32_t call_count = 0;
         int32_t op_errno = 0;
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
-
         priv = this->private;
 
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
 
         call_count = local->call_count;
+	if (!call_count) {
+		op_errno = ENOTCONN;
+		goto out;
+	}
 
         for (i = 0; i < priv->child_count; i++) {
                 if (local->child_up[i]) {
@@ -3187,18 +2736,18 @@ afr_entrylk (call_frame_t *frame, xlator_t *this,
                 }
         }
 
-        ret = 0;
+	return 0;
 out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (entrylk, frame, -1, op_errno, NULL);
+	AFR_STACK_UNWIND (entrylk, frame, -1, op_errno, NULL);
+
         return 0;
 }
 
 
 
-int32_t
-afr_fentrylk_cbk (call_frame_t *frame, void *cookie,
-                  xlator_t *this, int32_t op_ret, int32_t op_errno, dict_t *xdata)
+int
+afr_fentrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		  int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
 {
         afr_local_t *local = NULL;
@@ -3225,33 +2774,28 @@ afr_fentrylk_cbk (call_frame_t *frame, void *cookie,
 }
 
 
-int32_t
-afr_fentrylk (call_frame_t *frame, xlator_t *this,
-              const char *volume, fd_t *fd,
-              const char *basename, entrylk_cmd cmd,
-              entrylk_type type, dict_t *xdata)
+int
+afr_fentrylk (call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+              const char *basename, entrylk_cmd cmd, entrylk_type type,
+	      dict_t *xdata)
 {
         afr_private_t *priv = NULL;
         afr_local_t *local  = NULL;
-        int ret = -1;
         int i = 0;
         int32_t call_count = 0;
-        int32_t op_errno = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+        int32_t op_errno = ENOMEM;
 
         priv = this->private;
 
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+        local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
 
         call_count = local->call_count;
+	if (!call_count) {
+		op_errno = ENOTCONN;
+		goto out;
+	}
 
         for (i = 0; i < priv->child_count; i++) {
                 if (local->child_up[i]) {
@@ -3265,82 +2809,85 @@ afr_fentrylk (call_frame_t *frame, xlator_t *this,
                 }
         }
 
-        ret = 0;
+	return 0;
 out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (fentrylk, frame, -1, op_errno, NULL);
+	AFR_STACK_UNWIND (fentrylk, frame, -1, op_errno, NULL);
+
         return 0;
 }
 
-int32_t
-afr_statfs_cbk (call_frame_t *frame, void *cookie,
-                xlator_t *this, int32_t op_ret, int32_t op_errno,
-                struct statvfs *statvfs, dict_t *xdata)
+
+int
+afr_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+		int op_errno, struct statvfs *statvfs, dict_t *xdata)
 {
         afr_local_t *local = NULL;
         int call_count = 0;
+	struct statvfs *buf = NULL;
 
         LOCK (&frame->lock);
         {
                 local = frame->local;
 
-                if (op_ret == 0) {
-                        local->op_ret   = op_ret;
-
-                        if (local->cont.statfs.buf_set) {
-                                if (statvfs->f_bavail < local->cont.statfs.buf.f_bavail)
-                                        local->cont.statfs.buf = *statvfs;
-                        } else {
-                                local->cont.statfs.buf = *statvfs;
-                                local->cont.statfs.buf_set = 1;
-                        }
-                }
-
-                if (op_ret == -1)
+                if (op_ret != 0) {
                         local->op_errno = op_errno;
-
+			goto unlock;
+		}
+
+		local->op_ret = op_ret;
+
+		buf = &local->cont.statfs.buf;
+		if (local->cont.statfs.buf_set) {
+			if (statvfs->f_bavail < buf->f_bavail) {
+				*buf = *statvfs;
+				if (xdata) {
+					if (local->xdata_rsp)
+						dict_unref (local->xdata_rsp);
+					local->xdata_rsp = dict_ref (xdata);
+				}
+			}
+		} else {
+			*buf = *statvfs;
+			local->cont.statfs.buf_set = 1;
+			if (xdata)
+				local->xdata_rsp = dict_ref (xdata);
+		}
         }
+unlock:
         UNLOCK (&frame->lock);
 
         call_count = afr_frame_return (frame);
 
         if (call_count == 0)
                 AFR_STACK_UNWIND (statfs, frame, local->op_ret, local->op_errno,
-                                  &local->cont.statfs.buf, xdata);
+                                  &local->cont.statfs.buf, local->xdata_rsp);
 
         return 0;
 }
 
 
-int32_t
-afr_statfs (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, dict_t *xdata)
+int
+afr_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        afr_private_t *  priv        = NULL;
-        int              child_count = 0;
         afr_local_t   *  local       = NULL;
+	afr_private_t   *priv        = NULL;
         int              i           = 0;
-        int              ret = -1;
         int              call_count = 0;
-        int32_t          op_errno    = 0;
+        int32_t          op_errno    = ENOMEM;
 
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
-        VALIDATE_OR_GOTO (loc, out);
-
-        priv = this->private;
-        child_count = priv->child_count;
-
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
+	priv = this->private;
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
 
         call_count = local->call_count;
+	if (!call_count) {
+		op_errno = ENOTCONN;
+		goto out;
+	}
 
-        for (i = 0; i < child_count; i++) {
+        for (i = 0; i < priv->child_count; i++) {
                 if (local->child_up[i]) {
                         STACK_WIND (frame, afr_statfs_cbk,
                                     priv->children[i],
@@ -3351,10 +2898,10 @@ afr_statfs (call_frame_t *frame, xlator_t *this,
                 }
         }
 
-        ret = 0;
+	return 0;
 out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (statfs, frame, -1, op_errno, NULL, NULL);
+	AFR_STACK_UNWIND (statfs, frame, -1, op_errno, NULL, NULL);
+
         return 0;
 }
 
@@ -3463,21 +3010,6 @@ afr_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 AFR_STACK_UNWIND (lk, frame, -1, ENOTCONN,
                                   &local->cont.lk.ret_flock, NULL);
         } else {
-                /* locking has succeeded on all nodes that are up */
-
-                /* temporarily
-                   ret = afr_mark_locked_nodes (this, local->fd,
-                   local->cont.lk.locked_nodes);
-                   if (ret)
-                   gf_log (this->name, GF_LOG_DEBUG,
-                   "Could not save locked nodes info in fdctx");
-
-                   ret = afr_save_locked_fd (this, local->fd);
-                   if (ret)
-                   gf_log (this->name, GF_LOG_DEBUG,
-                   "Could not save locked fd");
-
-                */
                 AFR_STACK_UNWIND (lk, frame, local->op_ret, local->op_errno,
                                   &local->cont.lk.ret_flock, NULL);
         }
@@ -3493,20 +3025,12 @@ afr_lk (call_frame_t *frame, xlator_t *this,
         afr_private_t *priv = NULL;
         afr_local_t *local = NULL;
         int i = 0;
-        int32_t op_errno = 0;
-        int     ret      = -1;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+        int32_t op_errno = ENOMEM;
 
         priv = this->private;
 
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
+        local = AFR_FRAME_INIT (frame, op_errno);
+        if (!local)
                 goto out;
 
         local->cont.lk.locked_nodes = GF_CALLOC (priv->child_count,
@@ -3528,29 +3052,16 @@ afr_lk (call_frame_t *frame, xlator_t *this,
                            priv->children[i]->fops->lk,
                            fd, cmd, flock, xdata);
 
-        ret = 0;
+	return 0;
 out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (lk, frame, -1, op_errno, NULL, NULL);
+	AFR_STACK_UNWIND (lk, frame, -1, op_errno, NULL, NULL);
+
         return 0;
 }
 
 int
 afr_forget (xlator_t *this, inode_t *inode)
 {
-        uint64_t        ctx_addr = 0;
-        afr_inode_ctx_t *ctx     = NULL;
-
-        inode_ctx_get (inode, this, &ctx_addr);
-
-        if (!ctx_addr)
-                goto out;
-
-        ctx = (afr_inode_ctx_t *)(long)ctx_addr;
-        if (ctx->fresh_children)
-                GF_FREE (ctx->fresh_children);
-        GF_FREE (ctx);
-out:
         return 0;
 }
 
@@ -3570,7 +3081,6 @@ afr_priv_dump (xlator_t *this)
         snprintf(key_prefix, GF_DUMP_MAX_BUF_LEN, "%s.%s", this->type, this->name);
         gf_proc_dump_add_section(key_prefix);
         gf_proc_dump_write("child_count", "%u", priv->child_count);
-        gf_proc_dump_write("read_child_rr", "%u", priv->read_child_rr);
         for (i = 0; i < priv->child_count; i++) {
                 sprintf (key, "child_up[%d]", i);
                 gf_proc_dump_write(key, "%d", priv->child_up[i]);
@@ -3627,7 +3137,7 @@ afr_notify (xlator_t *this, int32_t event,
         int             idx                 = -1;
         int             ret                 = -1;
         int             call_psh            = 0;
-        int             up_child            = AFR_ALL_CHILDREN;
+        int             up_child            = -1;
         dict_t          *input              = NULL;
         dict_t          *output             = NULL;
 
@@ -3636,6 +3146,14 @@ afr_notify (xlator_t *this, int32_t event,
         if (!priv)
                 return 0;
 
+        /*
+         * We need to reset this in case children come up in "staggered"
+         * fashion, so that we discover a late-arriving local subvolume.  Note
+         * that we could end up issuing N lookups to the first subvolume, and
+         * O(N^2) overall, but N is small for AFR so it shouldn't be an issue.
+         */
+        priv->did_discovery = _gf_false;
+
         had_heard_from_all = 1;
         for (i = 0; i < priv->child_count; i++) {
                 if (!priv->last_event[i]) {
@@ -3671,6 +3189,7 @@ afr_notify (xlator_t *this, int32_t event,
                          */
                         if (priv->child_up[idx] != 1) {
                                 priv->up_count++;
+				priv->event_generation++;
                         }
                         priv->child_up[idx] = 1;
 
@@ -3710,6 +3229,7 @@ afr_notify (xlator_t *this, int32_t event,
                          */
                         if (priv->child_up[idx] == 1) {
                                 priv->down_count++;
+				priv->event_generation++;
                         }
                         priv->child_up[idx] = 0;
 
@@ -3742,6 +3262,10 @@ afr_notify (xlator_t *this, int32_t event,
         case GF_EVENT_TRANSLATOR_OP:
                 input = data;
                 output = data2;
+                if (!had_heard_from_all) {
+                        ret = -1;
+                        goto out;
+                }
                 ret = afr_xl_op (this, input, output);
                 goto out;
                 break;
@@ -3772,8 +3296,7 @@ afr_notify (xlator_t *this, int32_t event,
 
                 LOCK (&priv->lock);
                 {
-                        up_children = afr_up_children_count (priv->child_up,
-                                                             priv->child_count);
+                        up_children = AFR_COUNT (priv->child_up, priv->child_count);
                         for (i = 0; i < priv->child_count; i++) {
                                 if (priv->last_event[i] == GF_EVENT_CHILD_UP) {
                                         event = GF_EVENT_CHILD_UP;
@@ -3793,39 +3316,23 @@ afr_notify (xlator_t *this, int32_t event,
         ret = 0;
         if (propagate)
                 ret = default_notify (this, event, data);
-        if (call_psh && priv->shd.iamshd)
-                afr_proactive_self_heal ((void*) (long) up_child);
 
+        if (call_psh && priv->shd.iamshd) {
+                afr_selfheal_childup (this, up_child);
+	}
 out:
         return ret;
 }
 
-int
-afr_first_up_child (unsigned char *child_up, size_t child_count)
-{
-        int         ret      = -1;
-        int         i        = 0;
-
-        GF_ASSERT (child_up);
-
-        for (i = 0; i < child_count; i++) {
-                if (child_up[i]) {
-                        ret = i;
-                        break;
-                }
-        }
-
-        return ret;
-}
 
 int
 afr_local_init (afr_local_t *local, afr_private_t *priv, int32_t *op_errno)
 {
-        int     ret = -1;
-
         local->op_ret = -1;
         local->op_errno = EUCLEAN;
 
+	syncbarrier_init (&local->barrier);
+
         local->child_up = GF_CALLOC (priv->child_count,
                                      sizeof (*local->child_up),
                                      gf_afr_mt_char);
@@ -3837,27 +3344,42 @@ afr_local_init (afr_local_t *local, afr_private_t *priv, int32_t *op_errno)
 
         memcpy (local->child_up, priv->child_up,
                 sizeof (*local->child_up) * priv->child_count);
-        local->call_count = afr_up_children_count (local->child_up,
-                                                   priv->child_count);
+        local->call_count = AFR_COUNT (local->child_up, priv->child_count);
         if (local->call_count == 0) {
                 gf_log (THIS->name, GF_LOG_INFO, "no subvolumes up");
                 if (op_errno)
                         *op_errno = ENOTCONN;
                 goto out;
         }
+	local->event_generation = priv->event_generation;
 
-        local->child_errno = GF_CALLOC (priv->child_count,
-                                        sizeof (*local->child_errno),
-                                        gf_afr_mt_int32_t);
-        if (!local->child_errno) {
-                if (op_errno)
-                        *op_errno = ENOMEM;
-                goto out;
-        }
+	local->read_attempted = GF_CALLOC (priv->child_count, sizeof (char),
+					   gf_afr_mt_char);
+	if (!local->read_attempted) {
+		if (op_errno)
+			*op_errno = ENOMEM;
+		goto out;
+	}
 
-        ret = 0;
+	local->readable = GF_CALLOC (priv->child_count, sizeof (char),
+				     gf_afr_mt_char);
+	if (!local->readable) {
+		if (op_errno)
+			*op_errno = ENOMEM;
+		goto out;
+	}
+
+	local->replies = GF_CALLOC(priv->child_count, sizeof(*local->replies),
+				   gf_afr_mt_reply_t);
+	if (!local->replies) {
+		if (op_errno)
+			*op_errno = ENOMEM;
+		goto out;
+	}
+
+	return 0;
 out:
-        return ret;
+        return -1;
 }
 
 int
@@ -3866,16 +3388,6 @@ afr_internal_lock_init (afr_internal_lock_t *lk, size_t child_count,
 {
         int             ret = -ENOMEM;
 
-        lk->inode_locked_nodes = GF_CALLOC (sizeof (*lk->inode_locked_nodes),
-                                            child_count, gf_afr_mt_char);
-        if (NULL == lk->inode_locked_nodes)
-                goto out;
-
-        lk->entry_locked_nodes = GF_CALLOC (sizeof (*lk->entry_locked_nodes),
-                                            child_count, gf_afr_mt_char);
-        if (NULL == lk->entry_locked_nodes)
-                goto out;
-
         lk->locked_nodes = GF_CALLOC (sizeof (*lk->locked_nodes),
                                       child_count, gf_afr_mt_char);
         if (NULL == lk->locked_nodes)
@@ -3934,6 +3446,21 @@ out:
 }
 
 int
+afr_inodelk_init (afr_inodelk_t *lk, char *dom, size_t child_count)
+{
+        int             ret = -ENOMEM;
+
+        lk->domain = dom;
+        lk->locked_nodes = GF_CALLOC (sizeof (*lk->locked_nodes),
+                                      child_count, gf_afr_mt_char);
+        if (NULL == lk->locked_nodes)
+                goto out;
+        ret = 0;
+out:
+        return ret;
+}
+
+int
 afr_transaction_local_init (afr_local_t *local, xlator_t *this)
 {
         int            child_up_count = 0;
@@ -3946,14 +3473,20 @@ afr_transaction_local_init (afr_local_t *local, xlator_t *this)
         if (ret < 0)
                 goto out;
 
+        if ((local->transaction.type == AFR_DATA_TRANSACTION) ||
+            (local->transaction.type == AFR_METADATA_TRANSACTION)) {
+                ret = afr_inodelk_init (&local->internal_lock.inodelk[0],
+                                        this->name, priv->child_count);
+                if (ret < 0)
+                        goto out;
+        }
+
         ret = -ENOMEM;
-        child_up_count = afr_up_children_count (local->child_up,
-                                                priv->child_count);
+        child_up_count = AFR_COUNT (local->child_up, priv->child_count);
         if (priv->optimistic_change_log && child_up_count == priv->child_count)
                 local->optimistic_change_log = 1;
 
-        local->first_up_child = afr_first_up_child (local->child_up,
-                                                    priv->child_count);
+	local->pre_op_compat = priv->pre_op_compat;
 
         local->transaction.eager_lock =
                 GF_CALLOC (sizeof (*local->transaction.eager_lock),
@@ -3963,120 +3496,36 @@ afr_transaction_local_init (afr_local_t *local, xlator_t *this)
         if (!local->transaction.eager_lock)
                 goto out;
 
-        local->fresh_children = afr_children_create (priv->child_count);
-        if (!local->fresh_children)
-                goto out;
-
-        if (local->fd) {
-                local->fd_open_on = GF_CALLOC (sizeof (*local->fd_open_on),
-                                               priv->child_count,
-                                               gf_afr_mt_char);
-                if (!local->fd_open_on)
-                        goto out;
-        }
-
         local->transaction.pre_op = GF_CALLOC (sizeof (*local->transaction.pre_op),
                                                priv->child_count,
                                                gf_afr_mt_char);
         if (!local->transaction.pre_op)
                 goto out;
 
+        local->transaction.fop_subvols = GF_CALLOC (sizeof (*local->transaction.fop_subvols),
+						    priv->child_count,
+						    gf_afr_mt_char);
+        if (!local->transaction.fop_subvols)
+                goto out;
+
+        local->transaction.failed_subvols = GF_CALLOC (sizeof (*local->transaction.failed_subvols),
+						       priv->child_count,
+						       gf_afr_mt_char);
+        if (!local->transaction.failed_subvols)
+                goto out;
+
         local->pending = afr_matrix_create (priv->child_count,
                                             AFR_NUM_CHANGE_LOGS);
         if (!local->pending)
                 goto out;
 
-        local->transaction.child_errno =
-                GF_CALLOC (sizeof (*local->transaction.child_errno),
-                           priv->child_count,
-                           gf_afr_mt_int32_t);
-        local->transaction.erase_pending = 1;
+	INIT_LIST_HEAD (&local->transaction.eager_locked);
 
         ret = 0;
 out:
         return ret;
 }
 
-void
-afr_reset_children (int32_t *fresh_children, int32_t child_count)
-{
-        unsigned int i = 0;
-        for (i = 0; i < child_count; i++)
-                fresh_children[i] = -1;
-}
-
-int32_t*
-afr_children_create (int32_t child_count)
-{
-        int32_t           *children = NULL;
-        int               i               = 0;
-
-        GF_ASSERT (child_count > 0);
-
-        children = GF_CALLOC (child_count, sizeof (*children),
-                              gf_afr_mt_int32_t);
-        if (NULL == children)
-                goto out;
-        for (i = 0; i < child_count; i++)
-                children[i] = -1;
-out:
-        return children;
-}
-
-void
-afr_children_add_child (int32_t *children, int32_t child,
-                        int32_t child_count)
-{
-        gf_boolean_t child_found = _gf_false;
-        int          i               = 0;
-
-        for (i = 0; i < child_count; i++) {
-                if (children[i] == -1)
-                        break;
-                if (children[i] == child) {
-                        child_found = _gf_true;
-                        break;
-                }
-        }
-
-        if (!child_found) {
-                GF_ASSERT (i < child_count);
-                children[i] = child;
-        }
-}
-
-void
-afr_children_rm_child (int32_t *children, int32_t child, int32_t child_count)
-{
-        int          i = 0;
-
-        GF_ASSERT ((child >= 0) && (child < child_count));
-        for (i = 0; i < child_count; i++) {
-                if (children[i] == -1)
-                        break;
-                if (children[i] == child) {
-                        if (i != (child_count - 1))
-                                memmove (children + i, children + i + 1,
-                                         sizeof (*children)*(child_count - i - 1));
-                        children[child_count - 1] = -1;
-                        break;
-                }
-        }
-}
-
-int
-afr_get_children_count (int32_t *children, unsigned int child_count)
-{
-        int count = 0;
-        int i = 0;
-
-        for (i = 0; i < child_count; i++) {
-                if (children[i] == -1)
-                        break;
-                count++;
-        }
-        return count;
-}
 
 void
 afr_set_low_priority (call_frame_t *frame)
@@ -4084,38 +3533,6 @@ afr_set_low_priority (call_frame_t *frame)
         frame->root->pid = LOW_PRIO_PROC_PID;
 }
 
-int
-afr_child_fd_ctx_set (xlator_t *this, fd_t *fd, int32_t child,
-                      int flags)
-{
-        int             ret = 0;
-        uint64_t        ctx = 0;
-        afr_fd_ctx_t    *fd_ctx      = NULL;
-
-        GF_ASSERT (fd && fd->inode);
-        ret = afr_fd_ctx_set (this, fd);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "could not set fd ctx for fd=%p", fd);
-                goto out;
-        }
-
-        ret = fd_ctx_get (fd, this, &ctx);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "could not get fd ctx for fd=%p", fd);
-                goto out;
-        }
-
-        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
-        fd_ctx->opened_on[child] = AFR_FD_OPENED;
-        if (!IA_ISDIR (fd->inode->ia_type)) {
-                fd_ctx->flags            = flags;
-        }
-        ret = 0;
-out:
-        return ret;
-}
 
 gf_boolean_t
 afr_have_quorum (char *logname, afr_private_t *priv)
@@ -4162,23 +3579,6 @@ afr_priv_destroy (afr_private_t *priv)
         if (!priv)
                 goto out;
         inode_unref (priv->root_inode);
-        GF_FREE (priv->shd.pos);
-        GF_FREE (priv->shd.pending);
-        GF_FREE (priv->shd.inprogress);
-//        for (i = 0; i < priv->child_count; i++)
-//                if (priv->shd.timer && priv->shd.timer[i])
-//                        gf_timer_call_cancel (this->ctx, priv->shd.timer[i]);
-        GF_FREE (priv->shd.timer);
-
-        if (priv->shd.healed)
-                eh_destroy (priv->shd.healed);
-
-        if (priv->shd.heal_failed)
-                eh_destroy (priv->shd.heal_failed);
-
-        if (priv->shd.split_brain)
-                eh_destroy (priv->shd.split_brain);
-
         GF_FREE (priv->last_event);
         if (priv->pending_key) {
                 for (i = 0; i < priv->child_count; i++)
@@ -4188,8 +3588,7 @@ afr_priv_destroy (afr_private_t *priv)
         GF_FREE (priv->children);
         GF_FREE (priv->child_up);
         LOCK_DESTROY (&priv->lock);
-        LOCK_DESTROY (&priv->read_child_lock);
-        pthread_mutex_destroy (&priv->mutex);
+
         GF_FREE (priv);
 out:
         return;
@@ -4205,3 +3604,22 @@ xlator_subvolume_count (xlator_t *this)
                 i++;
         return i;
 }
+
+
+void
+afr_handle_open_fd_count (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t     *local = NULL;
+        afr_fd_ctx_t    *fd_ctx   = NULL;
+
+        local = frame->local;
+
+        if (!local->fd)
+		return;
+
+	fd_ctx = afr_fd_ctx_get (local->fd, this);
+	if (!fd_ctx)
+		return;
+
+	fd_ctx->open_fd_count = local->open_fd_count;
+}
diff --git a/xlators/cluster/afr/src/afr-dir-read.c b/xlators/cluster/afr/src/afr-dir-read.c
index b661cfa9c..fa1da3958 100644
--- a/xlators/cluster/afr/src/afr-dir-read.c
+++ b/xlators/cluster/afr/src/afr-dir-read.c
@@ -37,177 +37,7 @@
 #include "checksum.h"
 
 #include "afr.h"
-#include "afr-self-heal.h"
-#include "afr-self-heal-common.h"
-
-int
-afr_examine_dir_sh_unwind (call_frame_t *frame, xlator_t *this, int32_t op_ret,
-                           int32_t op_errno, int32_t sh_failed)
-{
-        afr_local_t *local  = NULL;
-
-        local = frame->local;
-
-        afr_set_opendir_done (this, local->fd->inode);
-
-        AFR_STACK_UNWIND (opendir, frame, local->op_ret,
-                          local->op_errno, local->fd, NULL);
-
-        return 0;
-}
-
-
-gf_boolean_t
-__checksums_differ (uint32_t *checksum, int child_count,
-                    unsigned char *child_up)
-{
-        int          ret            = _gf_false;
-        int          i              = 0;
-        uint32_t     cksum          = 0;
-        gf_boolean_t activate_check = _gf_false;
-
-        for (i = 0; i < child_count; i++) {
-                if (!child_up[i])
-                        continue;
-                if (_gf_false == activate_check) {
-                        cksum          = checksum[i];
-                        activate_check = _gf_true;
-                        continue;
-                }
-
-                if (cksum != checksum[i]) {
-                        ret = _gf_true;
-                        break;
-                }
-
-                cksum = checksum[i];
-        }
-
-        return ret;
-}
-
-
-int32_t
-afr_examine_dir_readdir_cbk (call_frame_t *frame, void *cookie,
-                             xlator_t *this, int32_t op_ret, int32_t op_errno,
-                             gf_dirent_t *entries, dict_t *xdata)
-{
-        afr_private_t *   priv        = NULL;
-        afr_local_t *     local       = NULL;
-        afr_self_heal_t * sh          = NULL;
-        gf_dirent_t *     entry       = NULL;
-        gf_dirent_t *     tmp         = NULL;
-        char              *reason     = NULL;
-        int               child_index = 0;
-        uint32_t          entry_cksum = 0;
-        int               call_count  = 0;
-        off_t             last_offset = 0;
-        inode_t           *inode      = NULL;
-
-        priv  = this->private;
-        local = frame->local;
-        sh    = &local->self_heal;
-        inode = local->fd->inode;
-
-        child_index = (long) cookie;
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%s: failed to do opendir on %s",
-                        local->loc.path, priv->children[child_index]->name);
-                local->op_ret = -1;
-                local->op_ret = op_errno;
-                goto out;
-        }
-
-        if (op_ret == 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "%s: no entries found in %s",
-                        local->loc.path, priv->children[child_index]->name);
-                goto out;
-        }
-
-        list_for_each_entry_safe (entry, tmp, &entries->list, list) {
-                entry_cksum = gf_rsync_weak_checksum ((unsigned char *)entry->d_name,
-                                                      strlen (entry->d_name));
-                local->cont.opendir.checksum[child_index] ^= entry_cksum;
-        }
-
-        list_for_each_entry (entry, &entries->list, list) {
-                last_offset = entry->d_off;
-        }
-
-        /* read more entries */
-
-        STACK_WIND_COOKIE (frame, afr_examine_dir_readdir_cbk,
-                           (void *) (long) child_index,
-                           priv->children[child_index],
-                           priv->children[child_index]->fops->readdir,
-                           local->fd, 131072, last_offset, NULL);
-
-        return 0;
-
-out:
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                if (__checksums_differ (local->cont.opendir.checksum,
-                                        priv->child_count,
-                                        local->child_up)) {
-
-                        sh->do_entry_self_heal  = _gf_true;
-                        sh->forced_merge          = _gf_true;
-
-                        reason = "checksums of directory differ";
-                        afr_launch_self_heal (frame, this, inode, _gf_false,
-                                              inode->ia_type, reason, NULL,
-                                              afr_examine_dir_sh_unwind);
-                } else {
-                        afr_set_opendir_done (this, inode);
-
-                        AFR_STACK_UNWIND (opendir, frame, local->op_ret,
-                                          local->op_errno, local->fd, NULL);
-                }
-        }
-
-        return 0;
-}
-
-
-int
-afr_examine_dir (call_frame_t *frame, xlator_t *this)
-{
-        afr_private_t * priv       = NULL;
-        afr_local_t *   local      = NULL;
-        int             i          = 0;
-        int             call_count = 0;
-
-        local = frame->local;
-        priv  = this->private;
-
-        local->cont.opendir.checksum = GF_CALLOC (priv->child_count,
-                                                  sizeof (*local->cont.opendir.checksum),
-                                                  gf_afr_mt_int32_t);
-
-        call_count = afr_up_children_count (local->child_up, priv->child_count);
-
-        local->call_count = call_count;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, afr_examine_dir_readdir_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->readdir,
-                                           local->fd, 131072, 0, NULL);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
-
-        return 0;
-}
+#include "afr-transaction.h"
 
 
 int32_t
@@ -215,112 +45,66 @@ afr_opendir_cbk (call_frame_t *frame, void *cookie,
                  xlator_t *this, int32_t op_ret, int32_t op_errno,
                  fd_t *fd, dict_t *xdata)
 {
-        afr_private_t *priv              = NULL;
         afr_local_t   *local             = NULL;
-        int32_t        up_children_count = 0;
-        int            ret               = -1;
         int            call_count        = -1;
         int32_t        child_index       = 0;
+	afr_fd_ctx_t  *fd_ctx = NULL;
 
-        priv  = this->private;
         local = frame->local;
+	fd_ctx = local->fd_ctx;
         child_index = (long) cookie;
 
-        up_children_count = afr_up_children_count (local->child_up,
-                                                   priv->child_count);
-
         LOCK (&frame->lock);
         {
-                if (op_ret >= 0) {
+                if (op_ret == -1) {
+                        local->op_errno = op_errno;
+			fd_ctx->opened_on[child_index] = AFR_FD_NOT_OPENED;
+                } else {
                         local->op_ret = op_ret;
-                        ret = afr_child_fd_ctx_set (this, fd, child_index, 0);
-                        if (ret) {
-                                local->op_ret = -1;
-                                local->op_errno = -ret;
-                                goto unlock;
-                        }
+			fd_ctx->opened_on[child_index] = AFR_FD_OPENED;
+			if (!local->xdata_rsp && xdata)
+				local->xdata_rsp = dict_ref (xdata);
                 }
-
-                local->op_errno = op_errno;
         }
-unlock:
         UNLOCK (&frame->lock);
 
         call_count = afr_frame_return (frame);
 
-        if (call_count == 0) {
-                if (local->op_ret != 0)
-                        goto out;
-
-                if (!afr_is_opendir_done (this, local->fd->inode) &&
-                    up_children_count > 1) {
-
-                        /*
-                         * This is the first opendir on this inode. We need
-                         * to check if the directory's entries are the same
-                         * on all subvolumes. This is needed in addition
-                         * to regular entry self-heal because the readdir
-                         * call is sent only to the first subvolume, and
-                         * thus files that exist only there will never be healed
-                         * otherwise (assuming changelog shows no anomalies).
-                         */
-
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "reading contents of directory %s looking for mismatch",
-                                local->loc.path);
-
-                        afr_examine_dir (frame, this);
-
-                } else {
-                        /* do the unwind */
-                        goto out;
-                }
-        }
-
-        return 0;
-
-out:
-        AFR_STACK_UNWIND (opendir, frame, local->op_ret,
-                          local->op_errno, local->fd, NULL);
-
+        if (call_count == 0)
+		AFR_STACK_UNWIND (opendir, frame, local->op_ret,
+				  local->op_errno, local->fd, NULL);
         return 0;
 }
 
 
-int32_t
-afr_opendir (call_frame_t *frame, xlator_t *this,
-             loc_t *loc, fd_t *fd)
+int
+afr_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd)
 {
         afr_private_t * priv        = NULL;
         afr_local_t   * local       = NULL;
-        int             child_count = 0;
         int             i           = 0;
-        int             ret         = -1;
         int             call_count  = -1;
-        int32_t         op_errno    = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+        int32_t         op_errno    = ENOMEM;
+	afr_fd_ctx_t *fd_ctx = NULL;
 
         priv = this->private;
 
-        child_count = priv->child_count;
-
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
+        local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+	fd_ctx = afr_fd_ctx_get (fd, this);
+	if (!fd_ctx)
+		goto out;
 
         loc_copy (&local->loc, loc);
 
         local->fd    = fd_ref (fd);
+	local->fd_ctx = fd_ctx;
 
         call_count = local->call_count;
 
-        for (i = 0; i < child_count; i++) {
+        for (i = 0; i < priv->child_count; i++) {
                 if (local->child_up[i]) {
                         STACK_WIND_COOKIE (frame, afr_opendir_cbk,
                                            (void*) (long) i,
@@ -333,382 +117,284 @@ afr_opendir (call_frame_t *frame, xlator_t *this,
                 }
         }
 
-        ret = 0;
+	return 0;
 out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (opendir, frame, -1, op_errno, fd, NULL);
-
+	AFR_STACK_UNWIND (opendir, frame, -1, op_errno, fd, NULL);
         return 0;
 }
 
 
-/**
- * Common algorithm for directory read calls:
- *
- * - Try the fop on the first child that is up
- * - if we have failed due to ENOTCONN:
- *     try the next child
- *
- * Applicable to: readdir
- */
-
+#define BACKEND_D_OFF_BITS 63
+#define PRESENT_D_OFF_BITS 63
 
-struct entry_name {
-        char *name;
-        struct list_head list;
-};
+#define ONE 1ULL
+#define MASK (~0ULL)
+#define PRESENT_MASK (MASK >> (64 - PRESENT_D_OFF_BITS))
+#define BACKEND_MASK (MASK >> (64 - BACKEND_D_OFF_BITS))
 
+#define TOP_BIT (ONE << (PRESENT_D_OFF_BITS - 1))
+#define SHIFT_BITS (max (0, (BACKEND_D_OFF_BITS - PRESENT_D_OFF_BITS + 1)))
 
-static gf_boolean_t
-remembered_name (const char *name, struct list_head *entries)
+static uint64_t
+afr_bits_for (uint64_t num)
 {
-        struct entry_name *e   = NULL;
-        gf_boolean_t       ret = _gf_false;
+	uint64_t bits = 0, ctrl = 1;
 
-        list_for_each_entry (e, entries, list) {
-                if (!strcmp (name, e->name)) {
-                        ret = _gf_true;
-                        goto out;
-                }
-        }
+	while (ctrl < num) {
+		ctrl *= 2;
+		bits ++;
+	}
 
-out:
-        return ret;
+	return bits;
 }
 
-
-static void
-afr_remember_entries (gf_dirent_t *entries, fd_t *fd)
+int
+afr_itransform (xlator_t *this, int subvol, uint64_t x, uint64_t *y_p)
 {
-        struct entry_name *n      = NULL;
-        gf_dirent_t       *entry  = NULL;
-        int                ret    = 0;
-        uint64_t           ctx    = 0;
-        afr_fd_ctx_t      *fd_ctx = NULL;
-
-        ret = fd_ctx_get (fd, THIS, &ctx);
-        if (ret < 0) {
-                gf_log (THIS->name, GF_LOG_INFO,
-                        "could not get fd ctx for fd=%p", fd);
-                return;
+        afr_private_t *conf = NULL;
+        int         cnt = 0;
+        int         max = 0;
+        uint64_t    y = 0;
+        uint64_t    hi_mask = 0;
+        uint64_t    off_mask = 0;
+        int         max_bits = 0;
+
+        if (x == ((uint64_t) -1)) {
+                y = (uint64_t) -1;
+                goto out;
         }
 
-        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
+        conf = this->private;
+        if (!conf)
+                goto out;
 
-        list_for_each_entry (entry, &entries->list, list) {
-                n = GF_CALLOC (1, sizeof (*n), gf_afr_mt_entry_name);
-                n->name = gf_strdup (entry->d_name);
-                INIT_LIST_HEAD (&n->list);
+        max = conf->child_count;
+        cnt = subvol;
 
-                list_add (&n->list, &fd_ctx->entries);
-        }
-}
+	if (max == 1) {
+		y = x;
+		goto out;
+	}
 
+        max_bits = afr_bits_for (max);
 
-static off_t
-afr_filter_entries (gf_dirent_t *entries, fd_t *fd)
-{
-        gf_dirent_t  *entry  = NULL;
-        gf_dirent_t  *tmp    = NULL;
-        int           ret    = 0;
-        uint64_t      ctx    = 0;
-        afr_fd_ctx_t *fd_ctx = NULL;
-        off_t         offset = 0;
-
-        ret = fd_ctx_get (fd, THIS, &ctx);
-        if (ret < 0) {
-                gf_log (THIS->name, GF_LOG_INFO,
-                        "could not get fd ctx for fd=%p", fd);
-                return -1;
-        }
-
-        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
-
-        list_for_each_entry_safe (entry, tmp, &entries->list, list) {
-                offset = entry->d_off;
+        hi_mask = ~(PRESENT_MASK >> (max_bits + 1));
 
-                if (remembered_name (entry->d_name, &fd_ctx->entries)) {
-                        list_del (&entry->list);
-                        GF_FREE (entry);
-                }
+        if (x & hi_mask) {
+                /* HUGE d_off */
+                off_mask = MASK << max_bits;
+                y = TOP_BIT | ((x >> SHIFT_BITS) & off_mask) | cnt;
+        } else {
+                /* small d_off */
+                y = ((x * max) + cnt);
         }
 
-        return offset;
+out:
+        if (y_p)
+                *y_p = y;
+
+        return 0;
 }
 
 
-static void
-afr_forget_entries (fd_t *fd)
+int
+afr_deitransform (xlator_t *this, uint64_t y, int *subvol_p,
+                  uint64_t *x_p)
 {
-        struct entry_name *entry  = NULL;
-        struct entry_name *tmp    = NULL;
-        int                ret    = 0;
-        uint64_t           ctx    = 0;
-        afr_fd_ctx_t      *fd_ctx = NULL;
-
-        ret = fd_ctx_get (fd, THIS, &ctx);
-        if (ret < 0) {
-                gf_log (THIS->name, GF_LOG_INFO,
-                        "could not get fd ctx for fd=%p", fd);
-                return;
-        }
-
-        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
+        afr_private_t *conf = NULL;
+        int         cnt = 0;
+        int         max = 0;
+        uint64_t    x = 0;
+        int         subvol = 0;
+        int         max_bits = 0;
+        uint64_t    off_mask = 0;
+        uint64_t    host_mask = 0;
+
+        if (!this->private)
+                return -1;
 
-        list_for_each_entry_safe (entry, tmp, &fd_ctx->entries, list) {
-                GF_FREE (entry->name);
-                list_del (&entry->list);
-                GF_FREE (entry);
-        }
-}
+        conf = this->private;
+        max = conf->child_count;
 
+	if (max == 1) {
+		x = y;
+		cnt = 0;
+		goto out;
+	}
 
-int32_t
-afr_readdir_cbk (call_frame_t *frame, void *cookie,
-                 xlator_t *this, int32_t op_ret, int32_t op_errno,
-                 gf_dirent_t *entries, dict_t *xdata)
-{
-        afr_local_t *   local       = NULL;
-        gf_dirent_t *   entry       = NULL;
-        gf_dirent_t *   tmp         = NULL;
+        if (y & TOP_BIT) {
+                /* HUGE d_off */
+                max_bits = afr_bits_for (max);
+                off_mask = (MASK << max_bits);
+                host_mask = ~(off_mask);
 
-        local = frame->local;
-
-        if (op_ret == -1)
-                goto out;
+                x = ((y & ~TOP_BIT) & off_mask) << SHIFT_BITS;
 
-        list_for_each_entry_safe (entry, tmp, &entries->list, list) {
-                if ((local->fd->inode == local->fd->inode->table->root)
-                    && !strcmp (entry->d_name, GF_REPLICATE_TRASH_DIR)) {
-                        list_del_init (&entry->list);
-                        GF_FREE (entry);
-                }
+                cnt = y & host_mask;
+	} else {
+                /* small d_off */
+                cnt = y % max;
+                x = y / max;
         }
 
 out:
-        AFR_STACK_UNWIND (readdir, frame, op_ret, op_errno, entries, NULL);
+        subvol = cnt;
+
+        if (subvol_p)
+                *subvol_p = subvol;
+
+        if (x_p)
+                *x_p = x;
 
         return 0;
 }
 
 
-int32_t
-afr_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
-                  dict_t *xdata)
+static void
+afr_readdir_transform_entries (gf_dirent_t *subvol_entries, int subvol,
+			       gf_dirent_t *entries, fd_t *fd)
 {
-        afr_private_t *  priv            = NULL;
-        afr_local_t *    local           = NULL;
-        xlator_t **      children        = NULL;
-        int32_t          next_call_child = -1;
-        int              ret             = 0;
-        gf_dirent_t *    entry           = NULL;
-        gf_dirent_t *    tmp             = NULL;
-        int32_t          *last_index     = NULL;
-        int32_t          read_child      = -1;
-        int32_t         *fresh_children   = NULL;
-        uint64_t         ctx             = 0;
-        afr_fd_ctx_t    *fd_ctx          = NULL;
-        off_t            offset          = 0;
-        int32_t         call_child       = -1;
-
-        priv     = this->private;
-        children = priv->children;
-
-        local = frame->local;
-
-        read_child = (long) cookie;
-        last_index = &local->cont.readdir.last_index;
-        fresh_children = local->fresh_children;
-
-        /* the value of the last_index changes if afr_next_call_child is
-         * called. So to find the call_child of this callback use last_index
-         * before the next_call_child call.
-         */
-        if (*last_index == -1)
-                call_child = read_child;
-        else
-                call_child = fresh_children[*last_index];
-
-        if (priv->strict_readdir) {
-                ret = fd_ctx_get (local->fd, this, &ctx);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "could not get fd ctx for fd=%p", local->fd);
-                        op_ret   = -1;
-                        op_errno = -ret;
-                        goto out;
+	afr_private_t *priv = NULL;
+        gf_dirent_t *entry = NULL;
+        gf_dirent_t *tmp = NULL;
+	unsigned char *data_readable = NULL;
+	unsigned char *metadata_readable = NULL;
+	int gen = 0;
+
+	priv = THIS->private;
+
+	data_readable = alloca0 (priv->child_count);
+	metadata_readable = alloca0 (priv->child_count);
+
+        list_for_each_entry_safe (entry, tmp, &subvol_entries->list, list) {
+                if (__is_root_gfid (fd->inode->gfid) &&
+                    !strcmp (entry->d_name, GF_REPLICATE_TRASH_DIR)) {
+			continue;
                 }
 
-                fd_ctx = (afr_fd_ctx_t *)(long) ctx;
+		list_del_init (&entry->list);
+		afr_itransform (THIS, subvol, entry->d_off, &entry->d_off);
+		list_add_tail (&entry->list, &entries->list);
 
-                if (op_ret == -1) {
-                        next_call_child = afr_next_call_child (fresh_children,
-                                                               local->child_up,
-                                                               priv->child_count,
-                                                               last_index,
-                                                               read_child);
-                        if (next_call_child < 0)
-                                goto out;
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "starting readdir afresh on child %d, offset %"PRId64,
-                                next_call_child, (uint64_t) 0);
-
-                        fd_ctx->failed_over = _gf_true;
-
-                        STACK_WIND_COOKIE (frame, afr_readdirp_cbk,
-                                           (void *) (long) read_child,
-                                           children[next_call_child],
-                                           children[next_call_child]->fops->readdirp,
-                                           local->fd,
-                                           local->cont.readdir.size, 0,
-                                           local->cont.readdir.dict);
-                        return 0;
-                }
-        }
+		if (entry->inode) {
+			gen = 0;
+			afr_inode_read_subvol_get (entry->inode, THIS,
+						   data_readable,
+						   metadata_readable, &gen);
 
-        if (op_ret != -1) {
-                list_for_each_entry_safe (entry, tmp, &entries->list, list) {
-                        if ((local->fd->inode == local->fd->inode->table->root)
-                            && !strcmp (entry->d_name, GF_REPLICATE_TRASH_DIR)) {
-                                list_del_init (&entry->list);
-                                GF_FREE (entry);
-                        }
-                }
-        }
+			if (gen != priv->event_generation ||
+				!data_readable[subvol] ||
+				!metadata_readable[subvol]) {
 
-        if (priv->strict_readdir) {
-                if (fd_ctx->failed_over) {
-                        if (list_empty (&entries->list)) {
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                        "no entries found");
-                                goto out;
-                        }
-
-                        offset = afr_filter_entries (entries, local->fd);
-
-                        afr_remember_entries (entries, local->fd);
-
-                        if (list_empty (&entries->list)) {
-                                /* All the entries we got were duplicate. We
-                                   shouldn't send an empty list now, because
-                                   that will make the application stop reading. So
-                                   try to get more entries */
-
-                                gf_log (this->name, GF_LOG_TRACE,
-                                        "trying to fetch non-duplicate entries "
-                                        "from offset %"PRId64", child %s",
-                                        offset, children[call_child]->name);
-
-                                STACK_WIND_COOKIE (frame, afr_readdirp_cbk,
-                                                   (void *) (long) read_child,
-                                                   children[call_child],
-                                                   children[call_child]->fops->readdirp,
-                                                   local->fd, local->cont.readdir.size, offset,
-                                                   local->cont.readdir.dict);
-                                return 0;
-                        }
-                } else {
-                        afr_remember_entries (entries, local->fd);
-                }
+				inode_unref (entry->inode);
+				entry->inode = NULL;
+			}
+		}
         }
-
-out:
-        AFR_STACK_UNWIND (readdirp, frame, op_ret, op_errno, entries, NULL);
-
-        return 0;
 }
 
+
 int32_t
-afr_do_readdir (call_frame_t *frame, xlator_t *this,
-                fd_t *fd, size_t size, off_t offset, int whichop, dict_t *dict)
+afr_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		 int32_t op_ret, int32_t op_errno, gf_dirent_t *subvol_entries,
+		 dict_t *xdata)
 {
-        afr_private_t *  priv       = NULL;
-        xlator_t **      children   = NULL;
-        int              call_child = 0;
-        afr_local_t     *local      = NULL;
-        uint64_t         ctx        = 0;
-        afr_fd_ctx_t    *fd_ctx     = NULL;
-        int              ret        = -1;
-        int32_t          op_errno   = 0;
-        uint64_t         read_child = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
-
-        priv     = this->private;
-        children = priv->children;
-
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
+        afr_local_t *local = NULL;
+	gf_dirent_t  entries;
+
+	INIT_LIST_HEAD (&entries.list);
+
         local = frame->local;
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+        if (op_ret < 0 && !local->cont.readdir.offset) {
+		/* failover only if this was first readdir, detected
+		   by offset == 0 */
+		local->op_ret = op_ret;
+		local->op_errno = op_errno;
 
-        local->fresh_children = afr_children_create (priv->child_count);
-        if (!local->fresh_children) {
-                op_errno = ENOMEM;
-                goto out;
-        }
+		afr_read_txn_continue (frame, this, (long) cookie);
+		return 0;
+	}
 
-        read_child = afr_inode_get_read_ctx (this, fd->inode,
-                                             local->fresh_children);
-        ret = afr_get_call_child (this, local->child_up, read_child,
-                                     local->fresh_children,
-                                     &call_child,
-                                     &local->cont.readdir.last_index);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
-        }
+	if (op_ret >= 0)
+		afr_readdir_transform_entries (subvol_entries, (long) cookie,
+					       &entries, local->fd);
 
-        local->fd                  = fd_ref (fd);
-        local->cont.readdir.size   = size;
-        local->cont.readdir.dict   = (dict)? dict_ref (dict) : NULL;
-
-        if (priv->strict_readdir) {
-                ret = fd_ctx_get (fd, this, &ctx);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "could not get fd ctx for fd=%p", fd);
-                        op_errno = -ret;
-                        goto out;
-                }
+        AFR_STACK_UNWIND (readdir, frame, op_ret, op_errno, &entries, xdata);
 
-                fd_ctx = (afr_fd_ctx_t *)(long) ctx;
+        return 0;
+}
 
-                if (fd_ctx->last_tried != call_child) {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "first up child has changed from %d to %d, "
-                                "restarting readdir from offset 0",
-                                fd_ctx->last_tried, call_child);
 
-                        fd_ctx->failed_over = _gf_true;
-                        offset = 0;
-                }
+int
+afr_readdir_wind (call_frame_t *frame, xlator_t *this, int subvol)
+{
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
 
-                fd_ctx->last_tried = call_child;
-        }
+	priv = this->private;
+	local = frame->local;
+
+	if (subvol == -1) {
+		AFR_STACK_UNWIND (readdir, frame, local->op_ret,
+				  local->op_errno, 0, 0);
+		return 0;
+	}
 
-        if (whichop == GF_FOP_READDIR)
+        if (local->op == GF_FOP_READDIR)
                 STACK_WIND_COOKIE (frame, afr_readdir_cbk,
-                                   (void *) (long) call_child,
-                                   children[call_child],
-                                   children[call_child]->fops->readdir, fd,
-                                   size, offset, dict);
+                                   (void *) (long) subvol,
+                                   priv->children[subvol],
+                                   priv->children[subvol]->fops->readdir,
+				   local->fd, local->cont.readdir.size,
+                                   local->cont.readdir.offset,
+				   local->xdata_req);
         else
-                STACK_WIND_COOKIE (frame, afr_readdirp_cbk,
-                                   (void *) (long) call_child,
-                                   children[call_child],
-                                   children[call_child]->fops->readdirp, fd,
-                                   size, offset, dict);
+                STACK_WIND_COOKIE (frame, afr_readdir_cbk,
+                                   (void *) (long) subvol,
+                                   priv->children[subvol],
+                                   priv->children[subvol]->fops->readdirp,
+				   local->fd, local->cont.readdir.size,
+				   local->cont.readdir.offset,
+				   local->xdata_req);
+	return 0;
+}
+
 
-        ret = 0;
+int
+afr_do_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+		off_t offset, int whichop, dict_t *dict)
+{
+        afr_local_t   *local     = NULL;
+        int32_t       op_errno   = 0;
+	int           subvol = -1;
+
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
+
+	local->op = whichop;
+        local->fd = fd_ref (fd);
+        local->cont.readdir.size = size;
+	local->cont.readdir.offset = offset;
+        local->xdata_req = (dict)? dict_ref (dict) : NULL;
+
+	if (offset == 0) {
+		/* First readdir has option of failing over and selecting
+		   an appropriate read subvolume */
+		afr_read_txn (frame, this, fd->inode, afr_readdir_wind,
+			      AFR_DATA_TRANSACTION);
+	} else {
+		/* But continued readdirs MUST stick to the same subvolume
+		   without an option to failover */
+		afr_deitransform (this, offset, &subvol,
+				  (uint64_t *)&local->cont.readdir.offset);
+		afr_readdir_wind (frame, this, subvol);
+	}
+
+        return 0;
 out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (readdir, frame, -1, op_errno, NULL, NULL);
+        AFR_STACK_UNWIND (readdir, frame, -1, op_errno, NULL, NULL);
         return 0;
 }
 
@@ -717,7 +403,8 @@ int32_t
 afr_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
              off_t offset, dict_t *xdata)
 {
-        afr_do_readdir (frame, this, fd, size, offset, GF_FOP_READDIR, xdata);
+	afr_do_readdir (frame, this, fd, size, offset, GF_FOP_READDIR, xdata);
+
         return 0;
 }
 
@@ -727,6 +414,7 @@ afr_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
               off_t offset, dict_t *dict)
 {
         afr_do_readdir (frame, this, fd, size, offset, GF_FOP_READDIRP, dict);
+
         return 0;
 }
 
@@ -734,7 +422,6 @@ afr_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
 int32_t
 afr_releasedir (xlator_t *this, fd_t *fd)
 {
-        afr_forget_entries (fd);
         afr_cleanup_fd_ctx (this, fd);
 
         return 0;
diff --git a/xlators/cluster/afr/src/afr-dir-write.c b/xlators/cluster/afr/src/afr-dir-write.c
index 46f5dceeb..465dde54f 100644
--- a/xlators/cluster/afr/src/afr-dir-write.c
+++ b/xlators/cluster/afr/src/afr-dir-write.c
@@ -34,10 +34,14 @@
 #include "common-utils.h"
 #include "compat-errno.h"
 #include "compat.h"
+#include "byte-order.h"
 
 #include "afr.h"
 #include "afr-transaction.h"
 
+void
+afr_mark_entry_pending_changelog (call_frame_t *frame, xlator_t *this);
+
 int
 afr_build_parent_loc (loc_t *parent, loc_t *child, int32_t *op_errno)
 {
@@ -56,49 +60,192 @@ afr_build_parent_loc (loc_t *parent, loc_t *child, int32_t *op_errno)
                         *op_errno = ENOMEM;
                 goto out;
         }
-        parent->path = dirname (child_path);
-        parent->inode  = inode_ref (child->parent);
-        uuid_copy (parent->gfid, child->pargfid);
+
+        parent->path = gf_strdup (dirname (child_path));
+        if (!parent->path) {
+                if (op_errno)
+                        *op_errno = ENOMEM;
+                goto out;
+        }
+
+        parent->inode = inode_ref (child->parent);
+	uuid_copy (parent->gfid, child->pargfid);
 
         ret = 0;
 out:
+        GF_FREE (child_path);
+
         return ret;
 }
 
-/* {{{ create */
 
-int
-afr_create_unwind (call_frame_t *frame, xlator_t *this)
+static void
+__afr_dir_write_finalize (call_frame_t *frame, xlator_t *this)
 {
-        call_frame_t *main_frame = NULL;
-        afr_local_t  *local = NULL;
-        struct iatt  *unwind_buf = NULL;
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	int inode_read_subvol = -1;
+	int parent_read_subvol = -1;
+	int parent2_read_subvol = -1;
+	int i = 0;
+
+	local = frame->local;
+	priv = this->private;
+
+	if (local->inode) {
+		afr_replies_interpret (frame, this, local->inode);
+		inode_read_subvol = afr_data_subvol_get (local->inode, this,
+							 NULL, NULL);
+	}
+	if (local->parent)
+		parent_read_subvol = afr_data_subvol_get (local->parent, this,
+							  NULL, NULL);
+	if (local->parent2)
+		parent2_read_subvol = afr_data_subvol_get (local->parent2, this,
+							   NULL, NULL);
+
+	local->op_ret = -1;
+	local->op_errno = afr_final_errno (local, priv);
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!local->replies[i].valid)
+			continue;
+		if (local->replies[i].op_ret < 0) {
+			if (local->inode)
+				afr_inode_read_subvol_reset (local->inode,
+							     this);
+			if (local->parent)
+				afr_inode_read_subvol_reset (local->parent,
+							     this);
+			if (local->parent2)
+				afr_inode_read_subvol_reset (local->parent2,
+							     this);
+			continue;
+		}
+
+		if (local->op_ret == -1) {
+			local->op_ret = local->replies[i].op_ret;
+			local->op_errno = local->replies[i].op_errno;
+
+			local->cont.dir_fop.buf =
+				local->replies[i].poststat;
+			local->cont.dir_fop.preparent =
+				local->replies[i].preparent;
+			local->cont.dir_fop.postparent =
+				local->replies[i].postparent;
+			local->cont.dir_fop.prenewparent =
+				local->replies[i].preparent2;
+			local->cont.dir_fop.postnewparent =
+				local->replies[i].postparent2;
+			if (local->replies[i].xdata)
+				local->xdata_rsp =
+					dict_ref (local->replies[i].xdata);
+			continue;
+		}
+
+		if (i == inode_read_subvol) {
+			local->cont.dir_fop.buf =
+				local->replies[i].poststat;
+			if (local->replies[i].xdata) {
+				if (local->xdata_rsp)
+					dict_unref (local->xdata_rsp);
+				local->xdata_rsp =
+					dict_ref (local->replies[i].xdata);
+			}
+		}
+
+		if (i == parent_read_subvol) {
+			local->cont.dir_fop.preparent =
+				local->replies[i].preparent;
+			local->cont.dir_fop.postparent =
+				local->replies[i].postparent;
+		}
+
+		if (i == parent2_read_subvol) {
+			local->cont.dir_fop.prenewparent =
+				local->replies[i].preparent2;
+			local->cont.dir_fop.postnewparent =
+				local->replies[i].postparent2;
+		}
+	}
+}
+
+
+static void
+__afr_dir_write_fill (call_frame_t *frame, xlator_t *this, int child_index,
+		      int op_ret, int op_errno, struct iatt *poststat,
+		      struct iatt *preparent, struct iatt *postparent,
+		      struct iatt *preparent2, struct iatt *postparent2,
+		      dict_t *xdata)
+{
+        afr_local_t *local = NULL;
+	afr_fd_ctx_t *fd_ctx = NULL;
 
         local = frame->local;
+	fd_ctx = local->fd_ctx;
+
+	local->replies[child_index].valid = 1;
+	local->replies[child_index].op_ret = op_ret;
+	local->replies[child_index].op_errno = op_errno;
+
+	if (op_ret >= 0) {
+		if (poststat)
+			local->replies[child_index].poststat = *poststat;
+		if (preparent)
+			local->replies[child_index].preparent = *preparent;
+		if (postparent)
+			local->replies[child_index].postparent = *postparent;
+		if (preparent2)
+			local->replies[child_index].preparent2 = *preparent2;
+		if (postparent2)
+			local->replies[child_index].postparent2 = *postparent2;
+		if (xdata)
+			local->replies[child_index].xdata = dict_ref (xdata);
+
+		if (fd_ctx)
+			fd_ctx->opened_on[child_index] = AFR_FD_OPENED;
+	} else {
+		if (op_errno != ENOTEMPTY)
+			afr_transaction_fop_failed (frame, this, child_index);
+		if (fd_ctx)
+			fd_ctx->opened_on[child_index] = AFR_FD_NOT_OPENED;
+	}
+
+        return;
+}
 
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame) {
-                        main_frame = local->transaction.main_frame;
-                }
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
-
-        if (main_frame) {
-                if (local->cont.create.read_child_buf.ia_ino) {
-                        unwind_buf = &local->cont.create.read_child_buf;
-                } else {
-                        unwind_buf = &local->cont.create.buf;
-                }
-
-                AFR_STACK_UNWIND (create, main_frame,
-                                  local->op_ret, local->op_errno,
-                                  local->cont.create.fd,
-                                  local->cont.create.inode,
-                                  unwind_buf, &local->cont.create.preparent,
-                                  &local->cont.create.postparent,
-                                  NULL);
+
+static int
+__afr_dir_write_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		     int op_ret, int op_errno, struct iatt *buf,
+		     struct iatt *preparent, struct iatt *postparent,
+		     struct iatt *preparent2, struct iatt *postparent2,
+                     dict_t *xdata)
+{
+        afr_local_t *local = NULL;
+        int child_index = (long) cookie;
+        int call_count = -1;
+
+        local = frame->local;
+
+	LOCK (&frame->lock);
+	{
+		__afr_dir_write_fill (frame, this, child_index, op_ret,
+				      op_errno, buf, preparent, postparent,
+				      preparent2, postparent2, xdata);
+	}
+	UNLOCK (&frame->lock);
+        call_count = afr_frame_return (frame);
+
+        if (call_count == 0) {
+		__afr_dir_write_finalize (frame, this);
+
+		if (afr_txn_nothing_failed (frame, this))
+			local->transaction.unwind (frame, this);
+
+		afr_mark_entry_pending_changelog (frame, this);
+
+                local->transaction.resume (frame, this);
         }
 
         return 0;
@@ -106,205 +253,236 @@ afr_create_unwind (call_frame_t *frame, xlator_t *this)
 
 
 int
-afr_create_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno,
-                     fd_t *fd, inode_t *inode, struct iatt *buf,
-                     struct iatt *preparent, struct iatt *postparent,
-                     dict_t *xdata)
+afr_mark_new_entry_changelog_cbk (call_frame_t *frame, void *cookie,
+				  xlator_t *this, int op_ret, int op_errno,
+                                  dict_t *xattr, dict_t *xdata)
 {
-        afr_local_t     *local = NULL;
-        afr_private_t   *priv  = NULL;
-        uint64_t        ctx = 0;
-        afr_fd_ctx_t    *fd_ctx = NULL;
-        int             ret = 0;
-        int             call_count = -1;
-        int             child_index = -1;
-        int32_t         *fresh_children = NULL;
+        int call_count = 0;
 
-        local = frame->local;
-        priv  = this->private;
+        call_count = afr_frame_return (frame);
 
-        child_index = (long) cookie;
+        if (call_count == 0)
+                AFR_STACK_DESTROY (frame);
 
-        LOCK (&frame->lock);
-        {
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
+        return 0;
+}
 
-                if (op_ret != -1) {
-                        local->op_ret = op_ret;
 
-                        ret = afr_fd_ctx_set (this, fd);
+void
+afr_mark_new_entry_changelog (call_frame_t *frame, xlator_t *this)
+{
+        call_frame_t  *new_frame  = NULL;
+        afr_local_t   *local      = NULL;
+        afr_local_t   *new_local  = NULL;
+        afr_private_t *priv       = NULL;
+        dict_t        *xattr      = NULL;
+        int32_t       **changelog = NULL;
+        int           i           = 0;
+	int           idx         = 0;
+        int           op_errno    = ENOMEM;
+	unsigned char *pending    = NULL;
+	int           call_count   = 0;
 
-                        if (ret < 0) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "could not set ctx on fd=%p", fd);
+        local = frame->local;
+        priv = this->private;
 
-                                local->op_ret   = -1;
-                                local->op_errno = -ret;
-                                goto unlock;
-                        }
+        new_frame = copy_frame (frame);
+        if (!new_frame)
+                goto out;
 
-                        ret = fd_ctx_get (fd, this, &ctx);
+	new_local = AFR_FRAME_INIT (new_frame, op_errno);
+	if (!new_local)
+		goto out;
 
-                        if (ret < 0) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "could not get fd ctx for fd=%p", fd);
-                                local->op_ret   = -1;
-                                local->op_errno = -ret;
-                                goto unlock;
-                        }
+        changelog = afr_matrix_create (priv->child_count, AFR_NUM_CHANGE_LOGS);
+        if (!changelog)
+                goto out;
 
-                        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
+        xattr = dict_new ();
+	if (!xattr)
+		goto out;
 
-                        fd_ctx->opened_on[child_index] = AFR_FD_OPENED;
-                        fd_ctx->flags                  = local->cont.create.flags;
+	idx = afr_index_for_transaction_type (AFR_DATA_TRANSACTION);
 
-                        if (local->success_count == 0)
-                                local->cont.create.buf        = *buf;
+	pending = alloca0 (priv->child_count);
 
-                        if (child_index == local->read_child_index) {
-                                local->cont.create.read_child_buf = *buf;
-                                local->cont.create.preparent      = *preparent;
-                                local->cont.create.postparent     = *postparent;
-                        }
+	for (i = 0; i < priv->child_count; i++) {
+		if (local->transaction.pre_op[i] &&
+		    !local->transaction.failed_subvols[i]) {
+			call_count ++;
+			continue;
+		}
 
-                        local->cont.create.inode = inode;
+		changelog[i][idx] = hton32(1);
+		pending[i] = 1;
+	}
 
-                        fresh_children = local->fresh_children;
-                        fresh_children[local->success_count] = child_index;
-                        local->success_count++;
-                }
+        new_local->pending = changelog;
+        uuid_copy (new_local->loc.gfid, local->cont.dir_fop.buf.ia_gfid);
+        new_local->loc.inode = inode_ref (local->inode);
 
-                local->op_errno = op_errno;
-        }
 
-unlock:
-        UNLOCK (&frame->lock);
+	afr_set_pending_dict (priv, xattr, changelog);
 
-        call_count = afr_frame_return (frame);
+        new_local->call_count = call_count;
 
-        if (call_count == 0) {
-                afr_set_read_ctx_from_policy (this, inode,
-                                              local->fresh_children,
-                                              local->read_child_index,
-                                              priv->read_child);
-                local->transaction.unwind (frame, this);
+        for (i = 0; i < priv->child_count; i++) {
+		if (pending[i])
+                        continue;
 
-                local->transaction.resume (frame, this);
+                STACK_WIND_COOKIE (new_frame, afr_mark_new_entry_changelog_cbk,
+                                   (void *) (long) i, priv->children[i],
+                                   priv->children[i]->fops->xattrop,
+                                   &new_local->loc, GF_XATTROP_ADD_ARRAY,
+                                   xattr, NULL);
+		if (!--call_count)
+			break;
         }
 
-        return 0;
+        new_frame = NULL;
+out:
+        if (new_frame)
+                AFR_STACK_DESTROY (new_frame);
+	if (xattr)
+		dict_unref (xattr);
+        return;
 }
 
 
-int
-afr_create_wind (call_frame_t *frame, xlator_t *this)
+void
+afr_mark_entry_pending_changelog (call_frame_t *frame, xlator_t *this)
 {
         afr_local_t *local = NULL;
         afr_private_t *priv = NULL;
-        int call_count = -1;
-        int i = 0;
+	int pre_op_count = 0;
+	int failed_count = 0;
 
         local = frame->local;
-        priv = this->private;
+        priv  = this->private;
 
-        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
-                                                     priv->child_count);
+        if (local->op_ret < 0)
+		return;
 
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
+	if (local->op != GF_FOP_CREATE && local->op != GF_FOP_MKNOD)
+		return;
 
-        local->call_count = call_count;
+	pre_op_count = AFR_COUNT (local->transaction.pre_op, priv->child_count);
+	failed_count = AFR_COUNT (local->transaction.failed_subvols,
+				  priv->child_count);
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i]) {
-                        STACK_WIND_COOKIE (frame, afr_create_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->create,
-                                           &local->loc,
-                                           local->cont.create.flags,
-                                           local->cont.create.mode,
-                                           local->umask,
-                                           local->cont.create.fd,
-                                           local->xdata_req);
-                        if (!--call_count)
-                                break;
-                }
-        }
+	if (pre_op_count == priv->child_count && !failed_count)
+		return;
 
-        return 0;
+        afr_mark_new_entry_changelog (frame, this);
+
+        return;
 }
 
 
+/* {{{ create */
+
 int
-afr_create_done (call_frame_t *frame, xlator_t *this)
+afr_create_unwind (call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t * local = NULL;
+        call_frame_t *main_frame = NULL;
+        afr_local_t  *local = NULL;
 
         local = frame->local;
 
-        local->transaction.unwind (frame, this);
+	main_frame = afr_transaction_detach_fop_frame (frame);
 
-        AFR_STACK_DESTROY (frame);
+        if (!main_frame)
+		return 0;
 
+	AFR_STACK_UNWIND (create, main_frame, local->op_ret, local->op_errno,
+			  local->cont.create.fd, local->inode,
+			  &local->cont.dir_fop.buf,
+			  &local->cont.dir_fop.preparent,
+			  &local->cont.dir_fop.postparent, local->xdata_rsp);
         return 0;
 }
 
 
 int
-afr_create (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, int32_t flags, mode_t mode,
-            mode_t umask, fd_t *fd, dict_t *params)
+afr_create_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno,
+                     fd_t *fd, inode_t *inode, struct iatt *buf,
+                     struct iatt *preparent, struct iatt *postparent,
+                     dict_t *xdata)
 {
-        afr_private_t  *priv  = NULL;
-        afr_local_t    *local = NULL;
-        call_frame_t   *transaction_frame = NULL;
-        int             ret = -1;
-        int             op_errno = 0;
+        return __afr_dir_write_cbk (frame, cookie, this, op_ret, op_errno, buf,
+				    preparent, postparent, NULL, NULL, xdata);
+}
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+
+int
+afr_create_wind (call_frame_t *frame, xlator_t *this, int subvol)
+{
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
+
+        local = frame->local;
+        priv = this->private;
+
+	STACK_WIND_COOKIE (frame, afr_create_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->create,
+			   &local->loc, local->cont.create.flags,
+			   local->cont.create.mode, local->umask,
+			   local->cont.create.fd, local->xdata_req);
+        return 0;
+}
+
+
+int
+afr_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+	    mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+{
+        afr_private_t           *priv                   = NULL;
+        afr_local_t             *local                  = NULL;
+        afr_internal_lock_t     *int_lock               = NULL;
+        call_frame_t            *transaction_frame      = NULL;
+        int                     ret                     = -1;
+        int                     op_errno                = ENOMEM;
 
         priv = this->private;
 
         QUORUM_CHECK(create,out);
 
         transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                op_errno = ENOMEM;
+        if (!transaction_frame)
                 goto out;
-        }
 
-        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
-        local = transaction_frame->local;
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+	local = AFR_FRAME_INIT (transaction_frame, op_errno);
+	if (!local)
+		goto out;
 
         loc_copy (&local->loc, loc);
 
-        LOCK (&priv->read_child_lock);
-        {
-                local->read_child_index = (++priv->read_child_rr)
-                        % (priv->child_count);
-        }
-        UNLOCK (&priv->read_child_lock);
+	local->fd_ctx = afr_fd_ctx_get (fd, this);
+	if (!local->fd_ctx)
+		goto out;
+
+	local->inode = inode_ref (loc->inode);
+	local->parent = inode_ref (loc->parent);
 
+        local->op                = GF_FOP_CREATE;
         local->cont.create.flags = flags;
         local->cont.create.mode  = mode;
         local->cont.create.fd    = fd_ref (fd);
         local->umask  = umask;
-        if (params)
-                local->xdata_req = dict_ref (params);
 
-        local->transaction.fop    = afr_create_wind;
-        local->transaction.done   = afr_create_done;
+        if (xdata)
+                local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
+
+	if (!local->xdata_req)
+		goto out;
+
+        local->transaction.wind   = afr_create_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
         local->transaction.unwind = afr_create_unwind;
 
         ret = afr_build_parent_loc (&local->transaction.parent_loc, loc,
@@ -314,18 +492,30 @@ afr_create (call_frame_t *frame, xlator_t *this,
 
         local->transaction.main_frame = frame;
         local->transaction.basename = AFR_BASENAME (loc->path);
+        int_lock = &local->internal_lock;
 
-        afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        int_lock->lockee_count = 0;
+        ret = afr_init_entry_lockee (&int_lock->lockee[0], local,
+                                     &local->transaction.parent_loc,
+                                     local->transaction.basename,
+                                     priv->child_count);
+        if (ret)
+                goto out;
 
-        ret = 0;
-out:
+        int_lock->lockee_count++;
+        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
         if (ret < 0) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (create, frame, -1, op_errno,
-                                  NULL, NULL, NULL, NULL, NULL, NULL);
+            op_errno = -ret;
+            goto out;
         }
 
+	return 0;
+out:
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
+
+	AFR_STACK_UNWIND (create, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+			  NULL, NULL);
         return 0;
 }
 
@@ -338,34 +528,17 @@ afr_mknod_unwind (call_frame_t *frame, xlator_t *this)
 {
         call_frame_t *main_frame = NULL;
         afr_local_t  *local = NULL;
-        struct iatt *unwind_buf = NULL;
 
         local = frame->local;
 
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame) {
-                        main_frame = local->transaction.main_frame;
-                }
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
-
-        if (main_frame) {
-                if (local->cont.mknod.read_child_buf.ia_ino) {
-                        unwind_buf = &local->cont.mknod.read_child_buf;
-                } else {
-                        unwind_buf = &local->cont.mknod.buf;
-                }
-
-                AFR_STACK_UNWIND (mknod, main_frame,
-                                  local->op_ret, local->op_errno,
-                                  local->cont.mknod.inode,
-                                  unwind_buf, &local->cont.mknod.preparent,
-                                  &local->cont.mknod.postparent,
-                                  NULL);
-        }
+	main_frame = afr_transaction_detach_fop_frame (frame);
+        if (!main_frame)
+		return 0;
 
+	AFR_STACK_UNWIND (mknod, main_frame, local->op_ret, local->op_errno,
+			  local->inode, &local->cont.dir_fop.buf,
+			  &local->cont.dir_fop.preparent,
+			  &local->cont.dir_fop.postparent, local->xdata_rsp);
         return 0;
 }
 
@@ -376,162 +549,72 @@ afr_mknod_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     struct iatt *buf, struct iatt *preparent,
                     struct iatt *postparent, dict_t *xdata)
 {
-        afr_local_t     *local          = NULL;
-        afr_private_t   *priv           = NULL;
-        int             call_count      = -1;
-        int             child_index     = -1;
-        int32_t         *fresh_children = NULL;
-
-        local = frame->local;
-        priv = this->private;
-
-        child_index = (long) cookie;
-
-        LOCK (&frame->lock);
-        {
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
-
-                if (op_ret != -1) {
-                        local->op_ret = op_ret;
-
-                        if (local->success_count == 0)
-                                local->cont.mknod.buf   = *buf;
-
-                        if (child_index == local->read_child_index) {
-                                local->cont.mknod.read_child_buf = *buf;
-                                local->cont.mknod.preparent      = *preparent;
-                                local->cont.mknod.postparent     = *postparent;
-                        }
-
-                        local->cont.mknod.inode = inode;
-
-                        fresh_children = local->fresh_children;
-                        fresh_children[local->success_count] = child_index;
-                        local->success_count++;
-                }
-
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                afr_set_read_ctx_from_policy (this, inode,
-                                              local->fresh_children,
-                                              local->read_child_index,
-                                              priv->read_child);
-                local->transaction.unwind (frame, this);
-
-                local->transaction.resume (frame, this);
-        }
-
-        return 0;
+	return __afr_dir_write_cbk (frame, cookie, this, op_ret, op_errno, buf,
+				    preparent, postparent, NULL, NULL, xdata);
 }
 
 
-int32_t
-afr_mknod_wind (call_frame_t *frame, xlator_t *this)
+int
+afr_mknod_wind (call_frame_t *frame, xlator_t *this, int subvol)
 {
         afr_local_t *local = NULL;
         afr_private_t *priv = NULL;
-        int call_count = -1;
-        int i = 0;
 
         local = frame->local;
         priv  = this->private;
 
-        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
-                                                     priv->child_count);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
-
-        local->call_count = call_count;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i]) {
-                        STACK_WIND_COOKIE (frame, afr_mknod_wind_cbk, (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->mknod,
-                                           &local->loc, local->cont.mknod.mode,
-                                           local->cont.mknod.dev,
-                                           local->umask,
-                                           local->xdata_req);
-                        if (!--call_count)
-                                break;
-                }
-        }
-
+	STACK_WIND_COOKIE (frame, afr_mknod_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->mknod,
+			   &local->loc, local->cont.mknod.mode,
+			   local->cont.mknod.dev, local->umask,
+			   local->xdata_req);
         return 0;
 }
 
-
-int
-afr_mknod_done (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t * local = NULL;
-
-        local = frame->local;
-
-        local->transaction.unwind (frame, this);
-        AFR_STACK_DESTROY (frame);
-
-        return 0;
-}
-
-
 int
 afr_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-           dev_t dev, mode_t umask, dict_t *params)
+           dev_t dev, mode_t umask, dict_t *xdata)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t  * transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+        afr_private_t           *priv                   = NULL;
+        afr_local_t             *local                  = NULL;
+        afr_internal_lock_t     *int_lock               = NULL;
+        call_frame_t            *transaction_frame      = NULL;
+        int                     ret                     = -1;
+        int                     op_errno                = ENOMEM;
 
         priv = this->private;
 
         QUORUM_CHECK(mknod,out);
 
         transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                op_errno = ENOMEM;
+        if (!transaction_frame)
                 goto out;
-        }
-
-        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
-        local = transaction_frame->local;
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+	local = AFR_FRAME_INIT (transaction_frame, op_errno);
+	if (!local)
+		goto out;
 
         loc_copy (&local->loc, loc);
+	local->inode = inode_ref (loc->inode);
+	local->parent = inode_ref (loc->parent);
 
-        LOCK (&priv->read_child_lock);
-        {
-                local->read_child_index = (++priv->read_child_rr)
-                        % (priv->child_count);
-        }
-        UNLOCK (&priv->read_child_lock);
-
+        local->op               = GF_FOP_MKNOD;
         local->cont.mknod.mode  = mode;
         local->cont.mknod.dev   = dev;
         local->umask = umask;
-        if (params)
-                local->xdata_req = dict_ref (params);
 
-        local->transaction.fop    = afr_mknod_wind;
-        local->transaction.done   = afr_mknod_done;
+        if (xdata)
+                local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
+
+	if (!local->xdata_req)
+		goto out;
+
+        local->transaction.wind   = afr_mknod_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
         local->transaction.unwind = afr_mknod_unwind;
 
         ret = afr_build_parent_loc (&local->transaction.parent_loc, loc,
@@ -541,18 +624,30 @@ afr_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
 
         local->transaction.main_frame = frame;
         local->transaction.basename = AFR_BASENAME (loc->path);
+        int_lock = &local->internal_lock;
 
-        afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        int_lock->lockee_count = 0;
+        ret = afr_init_entry_lockee (&int_lock->lockee[0], local,
+                                     &local->transaction.parent_loc,
+                                     local->transaction.basename,
+                                     priv->child_count);
+        if (ret)
+                goto out;
 
-        ret = 0;
-out:
+        int_lock->lockee_count++;
+        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
         if (ret < 0) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (mknod, frame, -1, op_errno,
-                                  NULL, NULL, NULL, NULL, NULL);
+		op_errno = -ret;
+		goto out;
         }
 
+	return 0;
+out:
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
+
+	AFR_STACK_UNWIND (mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+			  NULL);
         return 0;
 }
 
@@ -566,34 +661,17 @@ afr_mkdir_unwind (call_frame_t *frame, xlator_t *this)
 {
         call_frame_t *main_frame = NULL;
         afr_local_t  *local = NULL;
-        struct iatt *unwind_buf = NULL;
 
         local = frame->local;
 
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame) {
-                        main_frame = local->transaction.main_frame;
-                }
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
-
-        if (main_frame) {
-                if (local->cont.mkdir.read_child_buf.ia_ino) {
-                        unwind_buf = &local->cont.mkdir.read_child_buf;
-                } else {
-                        unwind_buf = &local->cont.mkdir.buf;
-                }
-
-                AFR_STACK_UNWIND (mkdir, main_frame,
-                                  local->op_ret, local->op_errno,
-                                  local->cont.mkdir.inode,
-                                  unwind_buf, &local->cont.mkdir.preparent,
-                                  &local->cont.mkdir.postparent,
-                                  NULL);
-        }
+	main_frame = afr_transaction_detach_fop_frame (frame);
+        if (!main_frame)
+		return 0;
 
+	AFR_STACK_UNWIND (mkdir, main_frame, local->op_ret, local->op_errno,
+			  local->inode, &local->cont.dir_fop.buf,
+			  &local->cont.dir_fop.preparent,
+			  &local->cont.dir_fop.postparent, local->xdata_rsp);
         return 0;
 }
 
@@ -604,162 +682,71 @@ afr_mkdir_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     struct iatt *buf, struct iatt *preparent,
                     struct iatt *postparent, dict_t *xdata)
 {
-        afr_local_t     *local          = NULL;
-        afr_private_t   *priv           = NULL;
-        int             call_count      = -1;
-        int             child_index     = -1;
-        int32_t         *fresh_children = NULL;
-
-        local = frame->local;
-        priv = this->private;
-
-        child_index = (long) cookie;
-
-        LOCK (&frame->lock);
-        {
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
-
-                if (op_ret != -1) {
-                        local->op_ret           = op_ret;
-
-                        if (local->success_count == 0)
-                                local->cont.mkdir.buf   = *buf;
-
-                        if (child_index == local->read_child_index) {
-                                local->cont.mkdir.read_child_buf = *buf;
-                                local->cont.mkdir.preparent      = *preparent;
-                                local->cont.mkdir.postparent     = *postparent;
-                        }
-
-                        local->cont.mkdir.inode = inode;
-
-                        fresh_children = local->fresh_children;
-                        fresh_children[local->success_count] = child_index;
-                        local->success_count++;
-                }
-
-                local->op_errno         = op_errno;
-        }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                afr_set_read_ctx_from_policy (this, inode,
-                                              local->fresh_children,
-                                              local->read_child_index,
-                                              priv->read_child);
-                local->transaction.unwind (frame, this);
-
-                local->transaction.resume (frame, this);
-        }
-
-        return 0;
+	return __afr_dir_write_cbk (frame, cookie, this, op_ret, op_errno, buf,
+				    preparent, postparent, NULL, NULL, xdata);
 }
 
 
 int
-afr_mkdir_wind (call_frame_t *frame, xlator_t *this)
+afr_mkdir_wind (call_frame_t *frame, xlator_t *this, int subvol)
 {
         afr_local_t *local = NULL;
         afr_private_t *priv = NULL;
-        int call_count = -1;
-        int i = 0;
 
         local = frame->local;
         priv  = this->private;
 
-        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
-                                                     priv->child_count);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
-
-        local->call_count = call_count;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i]) {
-                        STACK_WIND_COOKIE (frame, afr_mkdir_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->mkdir,
-                                           &local->loc, local->cont.mkdir.mode,
-                                           local->umask,
-                                           local->xdata_req);
-                        if (!--call_count)
-                                break;
-                }
-        }
-
-        return 0;
-}
-
-
-int
-afr_mkdir_done (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t * local = NULL;
-
-        local = frame->local;
-
-        local->transaction.unwind (frame, this);
-
-        AFR_STACK_DESTROY (frame);
-
+	STACK_WIND_COOKIE (frame, afr_mkdir_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->mkdir, &local->loc,
+			   local->cont.mkdir.mode, local->umask,
+			   local->xdata_req);
         return 0;
 }
 
 
 int
-afr_mkdir (call_frame_t *frame, xlator_t *this,
-           loc_t *loc, mode_t mode, mode_t umask, dict_t *params)
+afr_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+	   mode_t umask, dict_t *xdata)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t  * transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+        afr_private_t           *priv                   = NULL;
+        afr_local_t             *local                  = NULL;
+        afr_internal_lock_t     *int_lock               = NULL;
+        call_frame_t            *transaction_frame      = NULL;
+        int                     ret                     = -1;
+        int                     op_errno                = ENOMEM;
 
         priv = this->private;
 
         QUORUM_CHECK(mkdir,out);
 
         transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                op_errno = ENOMEM;
+        if (!transaction_frame)
                 goto out;
-        }
-
-        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
-        local = transaction_frame->local;
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+	local = AFR_FRAME_INIT (transaction_frame, op_errno);
+	if (!local)
+		goto out;
 
         loc_copy (&local->loc, loc);
-
-        LOCK (&priv->read_child_lock);
-        {
-                local->read_child_index = (++priv->read_child_rr)
-                        % (priv->child_count);
-        }
-        UNLOCK (&priv->read_child_lock);
+	local->inode = inode_ref (loc->inode);
+	local->parent = inode_ref (loc->parent);
 
         local->cont.mkdir.mode  = mode;
         local->umask = umask;
-        if (params)
-                local->xdata_req = dict_ref (params);
 
-        local->transaction.fop    = afr_mkdir_wind;
-        local->transaction.done   = afr_mkdir_done;
+        if (xdata)
+                local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
+
+	if (!local->xdata_req)
+		goto out;
+
+        local->op = GF_FOP_MKDIR;
+        local->transaction.wind   = afr_mkdir_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
         local->transaction.unwind = afr_mkdir_unwind;
 
         ret = afr_build_parent_loc (&local->transaction.parent_loc, loc,
@@ -769,19 +756,30 @@ afr_mkdir (call_frame_t *frame, xlator_t *this,
 
         local->transaction.main_frame = frame;
         local->transaction.basename = AFR_BASENAME (loc->path);
+        int_lock = &local->internal_lock;
 
-        afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        int_lock->lockee_count = 0;
+        ret = afr_init_entry_lockee (&int_lock->lockee[0], local,
+                                     &local->transaction.parent_loc,
+                                     local->transaction.basename,
+                                     priv->child_count);
+        if (ret)
+                goto out;
 
-        ret = 0;
-out:
+        int_lock->lockee_count++;
+        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
         if (ret < 0) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-
-                AFR_STACK_UNWIND (mkdir, frame, -1, op_errno,
-                                  NULL, NULL, NULL, NULL, NULL);
+		op_errno = -ret;
+		goto out;
         }
 
+	return 0;
+out:
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
+
+	AFR_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+			  NULL);
         return 0;
 }
 
@@ -795,34 +793,17 @@ afr_link_unwind (call_frame_t *frame, xlator_t *this)
 {
         call_frame_t *main_frame = NULL;
         afr_local_t  *local = NULL;
-        struct iatt *unwind_buf = NULL;
 
         local = frame->local;
 
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame) {
-                        main_frame = local->transaction.main_frame;
-                }
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
-
-        if (main_frame) {
-                if (local->cont.link.read_child_buf.ia_ino) {
-                        unwind_buf = &local->cont.link.read_child_buf;
-                } else {
-                        unwind_buf = &local->cont.link.buf;
-                }
-
-                AFR_STACK_UNWIND (link, main_frame,
-                                  local->op_ret, local->op_errno,
-                                  local->cont.link.inode,
-                                  unwind_buf, &local->cont.link.preparent,
-                                  &local->cont.link.postparent,
-                                  NULL);
-        }
+	main_frame = afr_transaction_detach_fop_frame (frame);
+        if (!main_frame)
+		return 0;
 
+	AFR_STACK_UNWIND (link, main_frame, local->op_ret, local->op_errno,
+			  local->inode, &local->cont.dir_fop.buf,
+			  &local->cont.dir_fop.preparent,
+			  &local->cont.dir_fop.postparent, local->xdata_rsp);
         return 0;
 }
 
@@ -833,157 +814,70 @@ afr_link_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    struct iatt *buf, struct iatt *preparent,
                    struct iatt *postparent, dict_t *xdata)
 {
-        afr_local_t     *local          = NULL;
-        afr_private_t   *priv           = NULL;
-        int             call_count      = -1;
-        int             child_index     = -1;
-        int32_t         *fresh_children = NULL;
-
-        local = frame->local;
-        priv = this->private;
-
-        child_index = (long) cookie;
-
-        LOCK (&frame->lock);
-        {
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
-
-                if (op_ret != -1) {
-                        local->op_ret   = op_ret;
-
-                        if (local->success_count == 0) {
-                                local->cont.link.buf        = *buf;
-                        }
-
-                        if (child_index == local->read_child_index) {
-                                local->cont.link.read_child_buf = *buf;
-                                local->cont.link.preparent      = *preparent;
-                                local->cont.link.postparent     = *postparent;
-                        }
-
-                        fresh_children = local->fresh_children;
-                        fresh_children[local->success_count] = child_index;
-                        local->success_count++;
-                }
-
-		local->cont.link.inode = inode;
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                afr_set_read_ctx_from_policy (this, inode,
-                                              local->fresh_children,
-                                              local->read_child_index,
-                                              priv->read_child);
-                local->transaction.unwind (frame, this);
-
-                local->transaction.resume (frame, this);
-        }
-
-        return 0;
+        return __afr_dir_write_cbk (frame, cookie, this, op_ret, op_errno, buf,
+				    preparent, postparent, NULL, NULL, xdata);
 }
 
 
 int
-afr_link_wind (call_frame_t *frame, xlator_t *this)
+afr_link_wind (call_frame_t *frame, xlator_t *this, int subvol)
 {
         afr_local_t *local = NULL;
         afr_private_t *priv = NULL;
-        int call_count = -1;
-        int i = 0;
 
         local = frame->local;
         priv  = this->private;
 
-        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
-                                                     priv->child_count);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
-
-        local->call_count = call_count;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i]) {
-                        STACK_WIND_COOKIE (frame, afr_link_wind_cbk, (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->link,
-                                           &local->loc,
-                                           &local->newloc, local->xdata_req);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
-
+	STACK_WIND_COOKIE (frame, afr_link_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->link,
+			   &local->loc, &local->newloc, local->xdata_req);
         return 0;
 }
 
 
 int
-afr_link_done (call_frame_t *frame, xlator_t *this)
+afr_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+	  dict_t *xdata)
 {
-        afr_local_t * local = frame->local;
-
-        local->transaction.unwind (frame, this);
-
-        AFR_STACK_DESTROY (frame);
-
-        return 0;
-}
-
-
-int
-afr_link (call_frame_t *frame, xlator_t *this,
-          loc_t *oldloc, loc_t *newloc, dict_t *xdata)
-{
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t  * transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+        afr_private_t           *priv                   = NULL;
+        afr_local_t             *local                  = NULL;
+        afr_internal_lock_t     *int_lock               = NULL;
+        call_frame_t            *transaction_frame      = NULL;
+        int                     ret                     = -1;
+        int                     op_errno                = ENOMEM;
 
         priv = this->private;
 
         QUORUM_CHECK(link,out);
 
         transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                op_errno = ENOMEM;
+        if (!transaction_frame)
                 goto out;
-        }
-
-        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
-        local = transaction_frame->local;
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+	local = AFR_FRAME_INIT (transaction_frame, op_errno);
+	if (!local)
+		goto out;
 
         loc_copy (&local->loc,    oldloc);
         loc_copy (&local->newloc, newloc);
+
+	local->inode = inode_ref (oldloc->inode);
+	local->parent = inode_ref (newloc->parent);
+
         if (xdata)
-                local->xdata_req = dict_ref (xdata);
+                local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
 
-        LOCK (&priv->read_child_lock);
-        {
-                local->read_child_index = (++priv->read_child_rr)
-                        % (priv->child_count);
-        }
-        UNLOCK (&priv->read_child_lock);
+	if (!local->xdata_req)
+		goto out;
 
-        local->transaction.fop    = afr_link_wind;
-        local->transaction.done   = afr_link_done;
+        local->op = GF_FOP_LINK;
+
+        local->transaction.wind   = afr_link_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
         local->transaction.unwind = afr_link_unwind;
 
         ret = afr_build_parent_loc (&local->transaction.parent_loc, newloc,
@@ -993,18 +887,30 @@ afr_link (call_frame_t *frame, xlator_t *this,
 
         local->transaction.main_frame   = frame;
         local->transaction.basename     = AFR_BASENAME (newloc->path);
+        int_lock = &local->internal_lock;
 
-        afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        int_lock->lockee_count = 0;
+        ret = afr_init_entry_lockee (&int_lock->lockee[0], local,
+                                     &local->transaction.parent_loc,
+                                     local->transaction.basename,
+                                     priv->child_count);
+        if (ret)
+                goto out;
 
-        ret = 0;
-out:
+        int_lock->lockee_count++;
+        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
         if (ret < 0) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (link, frame, -1, op_errno,
-                                  NULL, NULL, NULL, NULL, NULL);
+		op_errno = -ret;
+		goto out;
         }
 
+	return 0;
+out:
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
+
+	AFR_STACK_UNWIND (link, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+			  NULL);
         return 0;
 }
 
@@ -1018,34 +924,17 @@ afr_symlink_unwind (call_frame_t *frame, xlator_t *this)
 {
         call_frame_t *main_frame = NULL;
         afr_local_t  *local = NULL;
-        struct iatt *unwind_buf = NULL;
 
         local = frame->local;
 
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame) {
-                        main_frame = local->transaction.main_frame;
-                }
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
-
-        if (main_frame) {
-                if (local->cont.symlink.read_child_buf.ia_ino) {
-                        unwind_buf = &local->cont.symlink.read_child_buf;
-                } else {
-                        unwind_buf = &local->cont.symlink.buf;
-                }
-
-                AFR_STACK_UNWIND (symlink, main_frame,
-                                  local->op_ret, local->op_errno,
-                                  local->cont.symlink.inode,
-                                  unwind_buf, &local->cont.symlink.preparent,
-                                  &local->cont.symlink.postparent,
-                                  NULL);
-        }
+	main_frame = afr_transaction_detach_fop_frame (frame);
+        if (!main_frame)
+		return 0;
 
+	AFR_STACK_UNWIND (symlink, main_frame, local->op_ret, local->op_errno,
+			  local->inode, &local->cont.dir_fop.buf,
+			  &local->cont.dir_fop.preparent,
+			  &local->cont.dir_fop.postparent, local->xdata_rsp);
         return 0;
 }
 
@@ -1056,163 +945,71 @@ afr_symlink_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                       struct iatt *buf, struct iatt *preparent,
                       struct iatt *postparent, dict_t *xdata)
 {
-        afr_local_t     *local          = NULL;
-        afr_private_t   *priv           = NULL;
-        int             call_count      = -1;
-        int             child_index     = -1;
-        int32_t         *fresh_children = NULL;
-
-        local = frame->local;
-        priv = this->private;
-
-        child_index = (long) cookie;
-
-        LOCK (&frame->lock);
-        {
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
-
-                if (op_ret != -1) {
-                        local->op_ret   = op_ret;
-
-                        if (local->success_count == 0)
-                                local->cont.symlink.buf        = *buf;
-
-                        if (child_index == local->read_child_index) {
-                                local->cont.symlink.read_child_buf = *buf;
-                                local->cont.symlink.preparent      = *preparent;
-                                local->cont.symlink.postparent     = *postparent;
-                        }
-
-                        local->cont.symlink.inode    = inode;
-
-                        fresh_children = local->fresh_children;
-                        fresh_children[local->success_count] = child_index;
-                        local->success_count++;
-                }
-
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                afr_set_read_ctx_from_policy (this, inode,
-                                              local->fresh_children,
-                                              local->read_child_index,
-                                              priv->read_child);
-                local->transaction.unwind (frame, this);
-
-                local->transaction.resume (frame, this);
-        }
-
-        return 0;
+        return __afr_dir_write_cbk (frame, cookie, this, op_ret, op_errno, buf,
+				    preparent, postparent, NULL, NULL, xdata);
 }
 
 
 int
-afr_symlink_wind (call_frame_t *frame, xlator_t *this)
+afr_symlink_wind (call_frame_t *frame, xlator_t *this, int subvol)
 {
         afr_local_t *local = NULL;
         afr_private_t *priv = NULL;
-        int call_count = -1;
-        int i = 0;
 
         local = frame->local;
         priv = this->private;
 
-        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
-                                                     priv->child_count);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
-
-        local->call_count = call_count;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i]) {
-                        STACK_WIND_COOKIE (frame, afr_symlink_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->symlink,
-                                           local->cont.symlink.linkpath,
-                                           &local->loc,
-                                           local->umask,
-                                           local->xdata_req);
-
-                        if (!--call_count)
-                                break;
-
-                }
-        }
-
-        return 0;
-}
-
-
-int
-afr_symlink_done (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t * local = frame->local;
-
-        local->transaction.unwind (frame, this);
-
-        AFR_STACK_DESTROY (frame);
-
+	STACK_WIND_COOKIE (frame, afr_symlink_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->symlink,
+			   local->cont.symlink.linkpath, &local->loc,
+			   local->umask, local->xdata_req);
         return 0;
 }
 
 
 int
-afr_symlink (call_frame_t *frame, xlator_t *this,
-             const char *linkpath, loc_t *loc, mode_t umask, dict_t *params)
+afr_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
+	     loc_t *loc, mode_t umask, dict_t *xdata)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t  * transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+        afr_private_t           *priv                   = NULL;
+        afr_local_t             *local                  = NULL;
+        afr_internal_lock_t     *int_lock               = NULL;
+        call_frame_t            *transaction_frame      = NULL;
+        int                     ret                     = -1;
+        int                     op_errno                = ENOMEM;
 
         priv = this->private;
 
         QUORUM_CHECK(symlink,out);
 
         transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                op_errno = ENOMEM;
+        if (!transaction_frame)
                 goto out;
-        }
 
-        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
-        local = transaction_frame->local;
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+	local = AFR_FRAME_INIT (transaction_frame, op_errno);
+	if (!local)
+		goto out;
 
         loc_copy (&local->loc, loc);
-
-        LOCK (&priv->read_child_lock);
-        {
-                local->read_child_index = (++priv->read_child_rr)
-                        % (priv->child_count);
-        }
-        UNLOCK (&priv->read_child_lock);
+	local->inode = inode_ref (loc->inode);
+	local->parent = inode_ref (loc->parent);
 
         local->cont.symlink.linkpath = gf_strdup (linkpath);
         local->umask = umask;
-        if (params)
-                local->xdata_req = dict_ref (params);
 
-        local->transaction.fop    = afr_symlink_wind;
-        local->transaction.done   = afr_symlink_done;
+        if (xdata)
+                local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
+
+	if (!local->xdata_req)
+		goto out;
+
+        local->op = GF_FOP_SYMLINK;
+        local->transaction.wind   = afr_symlink_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
         local->transaction.unwind = afr_symlink_unwind;
 
         ret = afr_build_parent_loc (&local->transaction.parent_loc, loc,
@@ -1222,18 +1019,30 @@ afr_symlink (call_frame_t *frame, xlator_t *this,
 
         local->transaction.main_frame   = frame;
         local->transaction.basename     = AFR_BASENAME (loc->path);
+        int_lock = &local->internal_lock;
 
-        afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        int_lock->lockee_count = 0;
+        ret = afr_init_entry_lockee (&int_lock->lockee[0], local,
+                                     &local->transaction.parent_loc,
+                                     local->transaction.basename,
+                                     priv->child_count);
+        if (ret)
+                goto out;
 
-        ret = 0;
-out:
-        if (ret  < 0) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (symlink, frame, -1, op_errno,
-                                  NULL, NULL, NULL, NULL, NULL);
+        int_lock->lockee_count++;
+        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        if (ret < 0) {
+		op_errno = -ret;
+		goto out;
         }
 
+	return 0;
+out:
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
+
+	AFR_STACK_UNWIND (symlink, frame, -1, op_errno, NULL, NULL, NULL,
+			  NULL, NULL);
         return 0;
 }
 
@@ -1246,36 +1055,19 @@ afr_rename_unwind (call_frame_t *frame, xlator_t *this)
 {
         call_frame_t *main_frame = NULL;
         afr_local_t  *local = NULL;
-        struct iatt *unwind_buf = NULL;
 
         local = frame->local;
 
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame) {
-                        main_frame = local->transaction.main_frame;
-                }
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
-
-        if (main_frame) {
-                if (local->cont.rename.read_child_buf.ia_ino) {
-                        unwind_buf = &local->cont.rename.read_child_buf;
-                } else {
-                        unwind_buf = &local->cont.rename.buf;
-                }
-
-                AFR_STACK_UNWIND (rename, main_frame,
-                                  local->op_ret, local->op_errno,
-                                  unwind_buf,
-                                  &local->cont.rename.preoldparent,
-                                  &local->cont.rename.postoldparent,
-                                  &local->cont.rename.prenewparent,
-                                  &local->cont.rename.postnewparent,
-                                  NULL);
-        }
+	main_frame = afr_transaction_detach_fop_frame (frame);
+        if (!main_frame)
+		return 0;
 
+	AFR_STACK_UNWIND (rename, main_frame, local->op_ret, local->op_errno,
+			  &local->cont.dir_fop.buf,
+			  &local->cont.dir_fop.preparent,
+			  &local->cont.dir_fop.postparent,
+			  &local->cont.dir_fop.prenewparent,
+			  &local->cont.dir_fop.postnewparent, local->xdata_rsp);
         return 0;
 }
 
@@ -1287,144 +1079,72 @@ afr_rename_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      struct iatt *prenewparent, struct iatt *postnewparent,
                      dict_t *xdata)
 {
-        afr_local_t *   local = NULL;
-        int call_count = -1;
-        int child_index = -1;
-
-        local = frame->local;
-
-        child_index = (long) cookie;
-
-        LOCK (&frame->lock);
-        {
-                if (afr_fop_failed (op_ret, op_errno) && op_errno != ENOTEMPTY)
-                        afr_transaction_fop_failed (frame, this, child_index);
-
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-
-                                if (buf) {
-                                        local->cont.rename.buf = *buf;
-                                }
-
-                                local->success_count++;
-                        }
-
-                        if (child_index == local->read_child_index) {
-                                local->cont.rename.read_child_buf = *buf;
-
-                                local->cont.rename.preoldparent  = *preoldparent;
-                                local->cont.rename.postoldparent = *postoldparent;
-                                local->cont.rename.prenewparent  = *prenewparent;
-                                local->cont.rename.postnewparent = *postnewparent;
-                        }
-                }
-
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                local->transaction.unwind (frame, this);
-                local->transaction.resume (frame, this);
-        }
-
-        return 0;
+        return __afr_dir_write_cbk (frame, cookie, this, op_ret, op_errno, buf,
+				    preoldparent, postoldparent, prenewparent,
+				    postnewparent, xdata);
 }
 
 
-int32_t
-afr_rename_wind (call_frame_t *frame, xlator_t *this)
+int
+afr_rename_wind (call_frame_t *frame, xlator_t *this, int subvol)
 {
         afr_local_t *local = NULL;
         afr_private_t *priv = NULL;
-        int call_count = -1;
-        int i = 0;
 
-        local = frame->local;
-        priv = this->private;
-
-        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
-                                                     priv->child_count);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
-
-        local->call_count = call_count;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i]) {
-                        STACK_WIND_COOKIE (frame, afr_rename_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->rename,
-                                           &local->loc,
-                                           &local->newloc, NULL);
-                        if (!--call_count)
-                                break;
-                }
-        }
-
-        return 0;
-}
-
-
-int
-afr_rename_done (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t * local = frame->local;
-
-        local->transaction.unwind (frame, this);
-
-        AFR_STACK_DESTROY (frame);
+	local = frame->local;
+	priv = this->private;
 
+	STACK_WIND_COOKIE (frame, afr_rename_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->rename,
+			   &local->loc, &local->newloc, local->xdata_req);
         return 0;
 }
 
 
 int
-afr_rename (call_frame_t *frame, xlator_t *this,
-            loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+afr_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+	    dict_t *xdata)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t  * transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+        afr_private_t           *priv                   = NULL;
+        afr_local_t             *local                  = NULL;
+        afr_internal_lock_t     *int_lock               = NULL;
+        call_frame_t            *transaction_frame      = NULL;
+        int                     ret                     = -1;
+        int                     op_errno                = ENOMEM;
+        int                     nlockee                 = 0;
 
         priv = this->private;
 
         QUORUM_CHECK(rename,out);
 
         transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
+        if (!transaction_frame)
                 op_errno = ENOMEM;
-                goto out;
-        }
 
-        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
-        local = transaction_frame->local;
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+	local = AFR_FRAME_INIT (transaction_frame, op_errno);
+	if (!local)
+		goto out;
 
         loc_copy (&local->loc,    oldloc);
         loc_copy (&local->newloc, newloc);
 
-        local->read_child_index = afr_inode_get_read_ctx (this, oldloc->inode, NULL);
+	local->inode = inode_ref (oldloc->inode);
+	local->parent = inode_ref (oldloc->parent);
+	local->parent2 = inode_ref (newloc->parent);
 
-        local->transaction.fop    = afr_rename_wind;
-        local->transaction.done   = afr_rename_done;
+        if (xdata)
+                local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
+
+	if (!local->xdata_req)
+		goto out;
+
+        local->op = GF_FOP_RENAME;
+        local->transaction.wind   = afr_rename_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
         local->transaction.unwind = afr_rename_unwind;
 
         ret = afr_build_parent_loc (&local->transaction.parent_loc, oldloc,
@@ -1439,19 +1159,52 @@ afr_rename (call_frame_t *frame, xlator_t *this,
         local->transaction.main_frame   = frame;
         local->transaction.basename     = AFR_BASENAME (oldloc->path);
         local->transaction.new_basename = AFR_BASENAME (newloc->path);
+        int_lock = &local->internal_lock;
 
-        afr_transaction (transaction_frame, this, AFR_ENTRY_RENAME_TRANSACTION);
+        int_lock->lockee_count = nlockee = 0;
+        ret = afr_init_entry_lockee (&int_lock->lockee[nlockee], local,
+                                     &local->transaction.new_parent_loc,
+                                     local->transaction.new_basename,
+                                     priv->child_count);
+        if (ret)
+                goto out;
 
-        ret = 0;
-out:
-        if (ret < 0) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
+        nlockee++;
+        ret = afr_init_entry_lockee (&int_lock->lockee[nlockee], local,
+                                     &local->transaction.parent_loc,
+                                     local->transaction.basename,
+                                     priv->child_count);
+        if (ret)
+                goto out;
+
+        nlockee++;
+        if (local->newloc.inode && IA_ISDIR (local->newloc.inode->ia_type)) {
+                ret = afr_init_entry_lockee (&int_lock->lockee[nlockee], local,
+                                             &local->newloc,
+                                             NULL,
+                                             priv->child_count);
+                if (ret)
+                        goto out;
+
+                nlockee++;
+        }
+        qsort (int_lock->lockee, nlockee, sizeof (*int_lock->lockee),
+               afr_entry_lockee_cmp);
+        int_lock->lockee_count = nlockee;
 
-                AFR_STACK_UNWIND (rename, frame, -1, op_errno,
-                                  NULL, NULL, NULL, NULL, NULL, NULL);
+        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_RENAME_TRANSACTION);
+        if (ret < 0) {
+		op_errno = -ret;
+		goto out;
         }
 
+	return 0;
+out:
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
+
+	AFR_STACK_UNWIND (rename, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+			  NULL, NULL);
         return 0;
 }
 
@@ -1467,23 +1220,13 @@ afr_unlink_unwind (call_frame_t *frame, xlator_t *this)
 
         local = frame->local;
 
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame) {
-                        main_frame = local->transaction.main_frame;
-                }
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
-
-        if (main_frame) {
-                AFR_STACK_UNWIND (unlink, main_frame,
-                                  local->op_ret, local->op_errno,
-                                  &local->cont.unlink.preparent,
-                                  &local->cont.unlink.postparent,
-                                  NULL);
-        }
+	main_frame = afr_transaction_detach_fop_frame (frame);
+        if (!main_frame)
+		return 0;
 
+	AFR_STACK_UNWIND (unlink, main_frame, local->op_ret, local->op_errno,
+			  &local->cont.dir_fop.preparent,
+			  &local->cont.dir_fop.postparent, local->xdata_rsp);
         return 0;
 }
 
@@ -1493,142 +1236,69 @@ afr_unlink_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno, struct iatt *preparent,
                      struct iatt *postparent, dict_t *xdata)
 {
-        afr_local_t *   local = NULL;
-        int call_count  = -1;
-        int child_index = (long) cookie;
-
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                if (child_index == local->read_child_index) {
-                        local->read_child_returned = _gf_true;
-                }
-
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
-
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret   = op_ret;
-                                local->cont.unlink.preparent  = *preparent;
-                                local->cont.unlink.postparent = *postparent;
-                        }
-
-                        if (child_index == local->read_child_index) {
-                                local->cont.unlink.preparent  = *preparent;
-                                local->cont.unlink.postparent = *postparent;
-                        }
-
-                        local->success_count++;
-                }
-
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                local->transaction.unwind (frame, this);
-
-                local->transaction.resume (frame, this);
-        }
-
-        return 0;
+        return __afr_dir_write_cbk (frame, cookie, this, op_ret, op_errno, NULL,
+				    preparent, postparent, NULL, NULL, xdata);
 }
 
 
-int32_t
-afr_unlink_wind (call_frame_t *frame, xlator_t *this)
+int
+afr_unlink_wind (call_frame_t *frame, xlator_t *this, int subvol)
 {
         afr_local_t *local = NULL;
         afr_private_t *priv = NULL;
-        int call_count = -1;
-        int i = 0;
 
         local = frame->local;
         priv  = this->private;
 
-        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
-                                                     priv->child_count);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
-
-        local->call_count = call_count;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i]) {
-                        STACK_WIND_COOKIE (frame, afr_unlink_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->unlink,
-                                           &local->loc, local->xflag,
-                                           local->xdata_req);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
-
+	STACK_WIND_COOKIE (frame, afr_unlink_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->unlink,
+			   &local->loc, local->xflag, local->xdata_req);
         return 0;
 }
 
 
-int32_t
-afr_unlink_done (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t * local = frame->local;
-
-        local->transaction.unwind (frame, this);
-
-        AFR_STACK_DESTROY (frame);
-
-        return 0;
-}
-
-
-int32_t
-afr_unlink (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, int xflag, dict_t *xdata)
+int
+afr_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+	    dict_t *xdata)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t  * transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+        afr_private_t           *priv                   = NULL;
+        afr_local_t             *local                  = NULL;
+        afr_internal_lock_t     *int_lock               = NULL;
+        call_frame_t            *transaction_frame      = NULL;
+        int                     ret                     = -1;
+        int                     op_errno                = ENOMEM;
 
         priv = this->private;
 
         QUORUM_CHECK(unlink,out);
 
         transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                op_errno = ENOMEM;
+        if (!transaction_frame)
                 goto out;
-        }
 
-        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
-        local = transaction_frame->local;
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+	local = AFR_FRAME_INIT (transaction_frame, op_errno);
+	if (!local)
+		goto out;
 
         loc_copy (&local->loc, loc);
         local->xflag = xflag;
+
+	local->inode = inode_ref (loc->inode);
+	local->parent = inode_ref (loc->parent);
+
         if (xdata)
-                local->xdata_req = dict_ref (xdata);
+                local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
 
-        local->transaction.fop    = afr_unlink_wind;
-        local->transaction.done   = afr_unlink_done;
+	if (!local->xdata_req)
+		goto out;
+
+        local->op = GF_FOP_UNLINK;
+        local->transaction.wind   = afr_unlink_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
         local->transaction.unwind = afr_unlink_unwind;
 
         ret = afr_build_parent_loc (&local->transaction.parent_loc, loc,
@@ -1638,18 +1308,29 @@ afr_unlink (call_frame_t *frame, xlator_t *this,
 
         local->transaction.main_frame = frame;
         local->transaction.basename = AFR_BASENAME (loc->path);
+        int_lock = &local->internal_lock;
 
-        afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        int_lock->lockee_count = 0;
+        ret = afr_init_entry_lockee (&int_lock->lockee[0], local,
+                                     &local->transaction.parent_loc,
+                                     local->transaction.basename,
+                                     priv->child_count);
+        if (ret)
+                goto out;
 
-        ret = 0;
-out:
+        int_lock->lockee_count++;
+        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
         if (ret < 0) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (unlink, frame, -1, op_errno,
-                                  NULL, NULL, NULL);
+		op_errno = -ret;
+		goto out;
         }
 
+	return 0;
+out:
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
+
+	AFR_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
@@ -1667,23 +1348,13 @@ afr_rmdir_unwind (call_frame_t *frame, xlator_t *this)
 
         local = frame->local;
 
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame) {
-                        main_frame = local->transaction.main_frame;
-                }
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
-
-        if (main_frame) {
-                AFR_STACK_UNWIND (rmdir, main_frame,
-                                  local->op_ret, local->op_errno,
-                                  &local->cont.rmdir.preparent,
-                                  &local->cont.rmdir.postparent,
-                                  NULL);
-        }
+	main_frame = afr_transaction_detach_fop_frame (frame);
+	if (!main_frame)
+		return 0;
 
+	AFR_STACK_UNWIND (rmdir, main_frame, local->op_ret, local->op_errno,
+			  &local->cont.dir_fop.preparent,
+			  &local->cont.dir_fop.postparent, local->xdata_rsp);
         return 0;
 }
 
@@ -1693,141 +1364,71 @@ afr_rmdir_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, struct iatt *preparent,
                     struct iatt *postparent, dict_t *xdata)
 {
-        afr_local_t *   local = NULL;
-        int call_count  = -1;
-        int child_index = (long) cookie;
-        int read_child  = 0;
-
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                if (child_index == read_child) {
-                        local->read_child_returned = _gf_true;
-                }
-
-                if (afr_fop_failed (op_ret, op_errno) && (op_errno != ENOTEMPTY))
-                        afr_transaction_fop_failed (frame, this, child_index);
-
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                                local->cont.rmdir.preparent  = *preparent;
-                                local->cont.rmdir.postparent = *postparent;
-
-                        }
-
-                        if (child_index == read_child) {
-                                local->cont.rmdir.preparent  = *preparent;
-                                local->cont.rmdir.postparent = *postparent;
-                        }
-
-                        local->success_count++;
-                }
-
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                local->transaction.unwind (frame, this);
-                local->transaction.resume (frame, this);
-        }
-
-        return 0;
+        return __afr_dir_write_cbk (frame, cookie, this, op_ret, op_errno, NULL,
+				    preparent, postparent, NULL, NULL, xdata);
 }
 
 
 int
-afr_rmdir_wind (call_frame_t *frame, xlator_t *this)
+afr_rmdir_wind (call_frame_t *frame, xlator_t *this, int subvol)
 {
         afr_local_t *local = NULL;
         afr_private_t *priv = NULL;
-        int call_count = -1;
-        int i = 0;
 
         local = frame->local;
         priv  = this->private;
 
-        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
-                                                     priv->child_count);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
-
-        local->call_count = call_count;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i]) {
-                        STACK_WIND_COOKIE (frame, afr_rmdir_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->rmdir,
-                                           &local->loc, local->cont.rmdir.flags,
-                                           NULL);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
-
-        return 0;
-}
-
-
-int
-afr_rmdir_done (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t * local = frame->local;
-
-        local->transaction.unwind (frame, this);
-
-        AFR_STACK_DESTROY (frame);
-
+	STACK_WIND_COOKIE (frame, afr_rmdir_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->rmdir,
+			   &local->loc, local->cont.rmdir.flags, local->xdata_req);
         return 0;
 }
 
 
 int
-afr_rmdir (call_frame_t *frame, xlator_t *this,
-           loc_t *loc, int flags, dict_t *xdata)
+afr_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+	   dict_t *xdata)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t  * transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+        afr_private_t           *priv                   = NULL;
+        afr_local_t             *local                  = NULL;
+        afr_internal_lock_t     *int_lock               = NULL;
+        call_frame_t            *transaction_frame      = NULL;
+        int                     ret                     = -1;
+        int                     op_errno                = ENOMEM;
+        int                     nlockee                 = 0;
 
         priv = this->private;
 
         QUORUM_CHECK(rmdir,out);
 
         transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                op_errno = ENOMEM;
+        if (!transaction_frame)
                 goto out;
-        }
 
-        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
-        local = transaction_frame->local;
+	local = AFR_FRAME_INIT (transaction_frame, op_errno);
+	if (!local)
+		goto out;
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
 
-        local->cont.rmdir.flags = flags;
         loc_copy (&local->loc, loc);
+	local->inode = inode_ref (loc->inode);
+	local->parent = inode_ref (loc->parent);
+
+        local->cont.rmdir.flags = flags;
+
+        if (xdata)
+                local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
+
+	if (!local->xdata_req)
+		goto out;
 
-        local->transaction.fop    = afr_rmdir_wind;
-        local->transaction.done   = afr_rmdir_done;
+        local->op = GF_FOP_RMDIR;
+        local->transaction.wind   = afr_rmdir_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
         local->transaction.unwind = afr_rmdir_unwind;
 
         ret = afr_build_parent_loc (&local->transaction.parent_loc, loc,
@@ -1837,17 +1438,41 @@ afr_rmdir (call_frame_t *frame, xlator_t *this,
 
         local->transaction.main_frame = frame;
         local->transaction.basename = AFR_BASENAME (loc->path);
+        int_lock = &local->internal_lock;
+
+        int_lock->lockee_count = nlockee = 0;
+        ret = afr_init_entry_lockee (&int_lock->lockee[nlockee], local,
+                                     &local->transaction.parent_loc,
+                                     local->transaction.basename,
+                                     priv->child_count);
+        if (ret)
+                goto out;
 
-        afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        nlockee++;
+        ret = afr_init_entry_lockee (&int_lock->lockee[nlockee], local,
+                                     &local->loc,
+                                     NULL,
+                                     priv->child_count);
+        if (ret)
+                goto out;
 
-        ret = 0;
-out:
+        nlockee++;
+        qsort (int_lock->lockee, nlockee, sizeof (*int_lock->lockee),
+               afr_entry_lockee_cmp);
+        int_lock->lockee_count = nlockee;
+
+        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
         if (ret < 0) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (rmdir, frame, -1, op_errno, NULL, NULL, NULL);
+		op_errno = -ret;
+		goto out;
         }
 
+	return 0;
+out:
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
+
+	AFR_STACK_UNWIND (rmdir, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
diff --git a/xlators/cluster/afr/src/afr-inode-read.c b/xlators/cluster/afr/src/afr-inode-read.c
index 40e5abd3a..01e078c13 100644
--- a/xlators/cluster/afr/src/afr-inode-read.c
+++ b/xlators/cluster/afr/src/afr-inode-read.c
@@ -35,237 +35,153 @@
 #include "compat-errno.h"
 #include "compat.h"
 
-/**
- * Common algorithm for inode read calls:
- *
- * - Try the fop on the first child that is up
- * - if we have failed due to ENOTCONN:
- *     try the next child
- *
- * Applicable to: access, stat, fstat, readlink, getxattr
- */
+#include "afr-transaction.h"
+
 
 /* {{{ access */
 
-int32_t
-afr_access_cbk (call_frame_t *frame, void *cookie,
-                xlator_t *this, int32_t op_ret, int32_t op_errno, dict_t *xdata)
+int
+afr_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		int op_ret, int op_errno, dict_t *xdata)
 {
-        afr_private_t * priv            = NULL;
-        afr_local_t *   local           = NULL;
-        xlator_t **     children        = NULL;
-        int             unwind          = 1;
-        int32_t         *last_index     = NULL;
-        int32_t         next_call_child = -1;
-        int32_t         read_child      = -1;
-        int32_t         *fresh_children  = NULL;
-
-        priv     = this->private;
-        children = priv->children;
+        afr_local_t     *local = NULL;
 
         local = frame->local;
 
-        read_child = (long) cookie;
-
-        if (op_ret == -1) {
-                last_index = &local->cont.access.last_index;
-                fresh_children = local->fresh_children;
-                next_call_child = afr_next_call_child (fresh_children,
-                                                       local->child_up,
-                                                       priv->child_count,
-                                                       last_index, read_child);
-                if (next_call_child < 0)
-                        goto out;
-
-                unwind = 0;
+	if (op_ret < 0) {
+		local->op_ret = op_ret;
+		local->op_errno = op_errno;
 
-                STACK_WIND_COOKIE (frame, afr_access_cbk,
-                                   (void *) (long) read_child,
-                                   children[next_call_child],
-                                   children[next_call_child]->fops->access,
-                                   &local->loc, local->cont.access.mask,
-                                   NULL);
-        }
+		afr_read_txn_continue (frame, this, (long) cookie);
+		return 0;
+	}
 
-out:
-        if (unwind) {
-                AFR_STACK_UNWIND (access, frame, op_ret, op_errno, xdata);
-        }
+	AFR_STACK_UNWIND (access, frame, op_ret, op_errno, xdata);
 
         return 0;
 }
 
 
-int32_t
-afr_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
-            dict_t *xdata)
+int
+afr_access_wind (call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_private_t   *priv      = NULL;
-        xlator_t        **children = NULL;
-        int             call_child = 0;
-        afr_local_t     *local     = NULL;
-        int32_t         op_errno   = 0;
-        int32_t         read_child = -1;
-        int             ret        = -1;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
-
-        priv     = this->private;
-        VALIDATE_OR_GOTO (priv->children, out);
-
-        children = priv->children;
-
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
-
-        local->fresh_children = afr_children_create (priv->child_count);
-        if (!local->fresh_children) {
-                op_errno = ENOMEM;
-                goto out;
-        }
+	afr_private_t *priv = NULL;
+	afr_local_t *local = NULL;
+
+	priv = this->private;
+	local = frame->local;
+
+	if (subvol == -1) {
+		AFR_STACK_UNWIND (access, frame, local->op_ret,
+				  local->op_errno, 0);
+		return 0;
+	}
+
+        STACK_WIND_COOKIE (frame, afr_access_cbk, (void *) (long) subvol,
+                           priv->children[subvol],
+                           priv->children[subvol]->fops->access,
+                           &local->loc, local->cont.access.mask,
+			   local->xdata_req);
+	return 0;
+}
 
+int
+afr_access (call_frame_t *frame, xlator_t *this, loc_t *loc,
+	    int mask, dict_t *xdata)
+{
+        afr_local_t *local = NULL;
+	int op_errno = 0;
 
-        read_child = afr_inode_get_read_ctx (this, loc->inode,
-                                             local->fresh_children);
-        ret = afr_get_call_child (this, local->child_up, read_child,
-                                     local->fresh_children,
-                                     &call_child,
-                                     &local->cont.access.last_index);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
-        }
+        local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
 
-        loc_copy (&local->loc, loc);
-        local->cont.access.mask = mask;
+	local->op = GF_FOP_ACCESS;
+	loc_copy (&local->loc, loc);
+	local->cont.access.mask = mask;
+	if (xdata)
+		local->xdata_req = dict_ref (xdata);
 
-        STACK_WIND_COOKIE (frame, afr_access_cbk,
-                           (void *) (long) call_child,
-                           children[call_child],
-                           children[call_child]->fops->access,
-                           loc, mask, xdata);
+	afr_read_txn (frame, this, loc->inode, afr_access_wind,
+		      AFR_METADATA_TRANSACTION);
 
-        ret = 0;
+	return 0;
 out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (access, frame, -1, op_errno, NULL);
+	AFR_STACK_UNWIND (access, frame, -1, op_errno, NULL);
+
         return 0;
 }
 
-
 /* }}} */
 
 /* {{{ stat */
 
-int32_t
+int
 afr_stat_cbk (call_frame_t *frame, void *cookie,
               xlator_t *this, int32_t op_ret, int32_t op_errno,
               struct iatt *buf, dict_t *xdata)
 {
-        afr_private_t * priv            = NULL;
-        afr_local_t *   local           = NULL;
-        xlator_t **     children        = NULL;
-        int             unwind          = 1;
-        int32_t         *last_index     = NULL;
-        int32_t         next_call_child = -1;
-        int32_t         read_child      = -1;
-        int32_t         *fresh_children  = NULL;
-
-        priv     = this->private;
-        children = priv->children;
-
-        read_child = (long) cookie;
+        afr_local_t     *local = NULL;
 
         local = frame->local;
 
-        if (op_ret == -1) {
-                last_index = &local->cont.stat.last_index;
-                fresh_children = local->fresh_children;
-                next_call_child = afr_next_call_child (fresh_children,
-                                                       local->child_up,
-                                                       priv->child_count,
-                                                       last_index, read_child);
-                if (next_call_child < 0)
-                        goto out;
-
-                unwind = 0;
+	if (op_ret < 0) {
+		local->op_ret = op_ret;
+		local->op_errno = op_errno;
 
-                STACK_WIND_COOKIE (frame, afr_stat_cbk,
-                                   (void *) (long) read_child,
-                                   children[next_call_child],
-                                   children[next_call_child]->fops->stat,
-                                   &local->loc, NULL);
-        }
+		afr_read_txn_continue (frame, this, (long) cookie);
+		return 0;
+	}
 
-out:
-        if (unwind) {
-                AFR_STACK_UNWIND (stat, frame, op_ret, op_errno, buf, xdata);
-        }
+	AFR_STACK_UNWIND (stat, frame, op_ret, op_errno, buf, xdata);
 
         return 0;
 }
 
 
-int32_t
-afr_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+int
+afr_stat_wind (call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_private_t   *priv      = NULL;
-        afr_local_t     *local     = NULL;
-        xlator_t        **children = NULL;
-        int             call_child = 0;
-        int32_t         op_errno   = 0;
-        int32_t         read_child = -1;
-        int             ret        = -1;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
-
-        priv     = this->private;
-        VALIDATE_OR_GOTO (priv->children, out);
-
-        children = priv->children;
+	afr_private_t *priv = NULL;
+	afr_local_t *local = NULL;
+
+	priv = this->private;
+	local = frame->local;
+
+	if (subvol == -1) {
+		AFR_STACK_UNWIND (stat, frame, local->op_ret, local->op_errno,
+				  0, 0);
+		return 0;
+	}
+
+        STACK_WIND_COOKIE (frame, afr_stat_cbk, (void *) (long) subvol,
+                           priv->children[subvol],
+                           priv->children[subvol]->fops->stat,
+                           &local->loc, local->xdata_req);
+	return 0;
+}
 
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
+int
+afr_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+        afr_local_t *local = NULL;
+	int op_errno = 0;
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+        local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
 
-        local->fresh_children = afr_children_create (priv->child_count);
-        if (!local->fresh_children) {
-                op_errno = ENOMEM;
-                goto out;
-        }
+	local->op = GF_FOP_STAT;
+	loc_copy (&local->loc, loc);
+	if (xdata)
+		local->xdata_req = dict_ref (xdata);
 
-        read_child = afr_inode_get_read_ctx (this, loc->inode,
-                                             local->fresh_children);
-        ret = afr_get_call_child (this, local->child_up, read_child,
-                                     local->fresh_children,
-                                     &call_child,
-                                     &local->cont.stat.last_index);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
-        }
-        loc_copy (&local->loc, loc);
-
-        STACK_WIND_COOKIE (frame, afr_stat_cbk, (void *) (long) call_child,
-                           children[call_child],
-                           children[call_child]->fops->stat,
-                           loc, xdata);
+	afr_read_txn (frame, this, loc->inode, afr_stat_wind,
+		      AFR_DATA_TRANSACTION);
 
-        ret = 0;
+	return 0;
 out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (stat, frame, -1, op_errno, NULL, NULL);
+	AFR_STACK_UNWIND (stat, frame, -1, op_errno, NULL, NULL);
 
         return 0;
 }
@@ -275,52 +191,49 @@ out:
 
 /* {{{ fstat */
 
-int32_t
+int
 afr_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                int32_t op_ret, int32_t op_errno, struct iatt *buf,
                dict_t *xdata)
 {
-        afr_private_t   *priv           = NULL;
-        afr_local_t     *local          = NULL;
-        xlator_t        **children      = NULL;
-        int             unwind          = 1;
-        int32_t         *last_index     = NULL;
-        int32_t         next_call_child = -1;
-        int32_t         read_child      = -1;
-        int32_t         *fresh_children  = NULL;
-
-        priv     = this->private;
-        children = priv->children;
+        afr_local_t     *local = NULL;
 
         local = frame->local;
 
-        read_child = (long) cookie;
+	if (op_ret < 0) {
+		local->op_ret = op_ret;
+		local->op_errno = op_errno;
 
-        if (op_ret == -1) {
-                last_index = &local->cont.fstat.last_index;
-                fresh_children = local->fresh_children;
-                next_call_child = afr_next_call_child (fresh_children,
-                                                       local->child_up,
-                                                       priv->child_count,
-                                                       last_index, read_child);
-                if (next_call_child < 0)
-                        goto out;
+		afr_read_txn_continue (frame, this, (long) cookie);
+		return 0;
+	}
 
-                unwind = 0;
+	AFR_STACK_UNWIND (fstat, frame, op_ret, op_errno, buf, xdata);
 
-                STACK_WIND_COOKIE (frame, afr_fstat_cbk,
-                                   (void *) (long) read_child,
-                                   children[next_call_child],
-                                   children[next_call_child]->fops->fstat,
-                                   local->fd, NULL);
-        }
+        return 0;
+}
 
-out:
-        if (unwind) {
-                AFR_STACK_UNWIND (fstat, frame, op_ret, op_errno, buf, xdata);
-        }
 
-        return 0;
+int
+afr_fstat_wind (call_frame_t *frame, xlator_t *this, int subvol)
+{
+	afr_private_t *priv = NULL;
+	afr_local_t *local = NULL;
+
+	priv = this->private;
+	local = frame->local;
+
+	if (subvol == -1) {
+		AFR_STACK_UNWIND (fstat, frame, local->op_ret, local->op_errno,
+				  0, 0);
+		return 0;
+	}
+
+        STACK_WIND_COOKIE (frame, afr_fstat_cbk, (void *) (long) subvol,
+                           priv->children[subvol],
+                           priv->children[subvol]->fops->fstat,
+                           local->fd, local->xdata_req);
+	return 0;
 }
 
 
@@ -328,69 +241,26 @@ int32_t
 afr_fstat (call_frame_t *frame, xlator_t *this,
            fd_t *fd, dict_t *xdata)
 {
-        afr_private_t   *priv      = NULL;
-        afr_local_t     *local     = NULL;
-        xlator_t        **children = NULL;
-        int             call_child = 0;
-        int32_t         op_errno   = 0;
-        int32_t         read_child = 0;
-        int             ret        = -1;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (fd, out);
-        VALIDATE_OR_GOTO (this->private, out);
-
-        priv     = this->private;
-        VALIDATE_OR_GOTO (priv->children, out);
+        afr_local_t *local = NULL;
+	int op_errno = 0;
 
-        children = priv->children;
+        local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
 
-        VALIDATE_OR_GOTO (fd->inode, out);
+	local->op = GF_FOP_FSTAT;
+	local->fd = fd_ref (fd);
+	if (xdata)
+		local->xdata_req = dict_ref (xdata);
 
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
+        afr_fix_open (fd, this);
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+	afr_read_txn (frame, this, fd->inode, afr_fstat_wind,
+		      AFR_DATA_TRANSACTION);
 
-        local->fresh_children = afr_children_create (priv->child_count);
-        if (!local->fresh_children) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        read_child = afr_inode_get_read_ctx (this, fd->inode,
-                                             local->fresh_children);
-
-
-
-        ret = afr_get_call_child (this, local->child_up, read_child,
-                                     local->fresh_children,
-                                     &call_child,
-                                     &local->cont.fstat.last_index);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
-        }
-
-        local->fd = fd_ref (fd);
-
-        ret = afr_open_fd_fix (frame, this, _gf_false);
-        if (ret) {
-                op_errno = -ret;
-                goto out;
-        }
-        STACK_WIND_COOKIE (frame, afr_fstat_cbk, (void *) (long) call_child,
-                           children[call_child],
-                           children[call_child]->fops->fstat,
-                           fd, xdata);
-
-        ret = 0;
+	return 0;
 out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (fstat, frame, -1, op_errno, NULL, NULL);
+	AFR_STACK_UNWIND (fstat, frame, -1, op_errno, NULL, NULL);
 
         return 0;
 }
@@ -399,115 +269,77 @@ out:
 
 /* {{{ readlink */
 
-int32_t
+int
 afr_readlink_cbk (call_frame_t *frame, void *cookie,
                   xlator_t *this, int32_t op_ret, int32_t op_errno,
                   const char *buf, struct iatt *sbuf, dict_t *xdata)
 {
-        afr_private_t * priv                  = NULL;
-        afr_local_t *   local                 = NULL;
-        xlator_t **     children              = NULL;
-        int             unwind                = 1;
-        int32_t         *last_index           = NULL;
-        int32_t         next_call_child       = -1;
-        int32_t         read_child            = -1;
-        int32_t         *fresh_children        = NULL;
+	afr_local_t *local = NULL;
 
-        priv     = this->private;
-        children = priv->children;
+	local = frame->local;
 
-        local = frame->local;
-
-        read_child = (long) cookie;
+	if (op_ret < 0) {
+		local->op_ret = -1;
+		local->op_errno = op_errno;
 
-        if (op_ret == -1) {
-                last_index = &local->cont.readlink.last_index;
-                fresh_children = local->fresh_children;
-                next_call_child = afr_next_call_child (fresh_children,
-                                                       local->child_up,
-                                                       priv->child_count,
-                                                       last_index, read_child);
-                if (next_call_child < 0)
-                        goto out;
-
-                unwind = 0;
-                STACK_WIND_COOKIE (frame, afr_readlink_cbk,
-                                   (void *) (long) read_child,
-                                   children[next_call_child],
-                                   children[next_call_child]->fops->readlink,
-                                   &local->loc,
-                                   local->cont.readlink.size, NULL);
-        }
+		afr_read_txn_continue (frame, this, (long) cookie);
+		return 0;
+	}
 
-out:
-        if (unwind) {
-                AFR_STACK_UNWIND (readlink, frame, op_ret, op_errno, buf, sbuf,
-                                  xdata);
-        }
+	AFR_STACK_UNWIND (readlink, frame, op_ret, op_errno,
+			  buf, sbuf, xdata);
+	return 0;
+}
 
-        return 0;
+int
+afr_readlink_wind (call_frame_t *frame, xlator_t *this, int subvol)
+{
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+
+	local = frame->local;
+	priv = this->private;
+
+	if (subvol == -1) {
+		AFR_STACK_UNWIND (readlink, frame, local->op_ret,
+				  local->op_errno, 0, 0, 0);
+		return 0;
+	}
+
+	STACK_WIND_COOKIE (frame, afr_readlink_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->readlink,
+			   &local->loc, local->cont.readlink.size,
+			   local->xdata_req);
+	return 0;
 }
 
 
-int32_t
+int
 afr_readlink (call_frame_t *frame, xlator_t *this,
               loc_t *loc, size_t size, dict_t *xdata)
 {
-        afr_private_t   *priv      = NULL;
-        xlator_t        **children = NULL;
-        int             call_child = 0;
-        afr_local_t     *local     = NULL;
+        afr_local_t   * local      = NULL;
         int32_t         op_errno   = 0;
-        int32_t         read_child = -1;
-        int             ret        = -1;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
-
-        priv     = this->private;
-        VALIDATE_OR_GOTO (priv->children, out);
-
-        children = priv->children;
-
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
 
-        local->fresh_children = afr_children_create (priv->child_count);
-        if (!local->fresh_children) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-        read_child = afr_inode_get_read_ctx (this, loc->inode,
-                                             local->fresh_children);
-        ret = afr_get_call_child (this, local->child_up, read_child,
-                                     local->fresh_children,
-                                     &call_child,
-                                     &local->cont.readlink.last_index);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
-        }
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
 
+	local->op = GF_FOP_READLINK;
         loc_copy (&local->loc, loc);
+        local->cont.readlink.size = size;
+	if (xdata)
+		local->xdata_req = dict_ref (xdata);
 
-        local->cont.readlink.size       = size;
+	afr_read_txn (frame, this, loc->inode, afr_readlink_wind,
+		      AFR_DATA_TRANSACTION);
 
-        STACK_WIND_COOKIE (frame, afr_readlink_cbk,
-                           (void *) (long) call_child,
-                           children[call_child],
-                           children[call_child]->fops->readlink,
-                           loc, size, xdata);
-
-        ret = 0;
-out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (readlink, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
+out:
+	AFR_STACK_UNWIND(readlink, frame, -1, op_errno, 0, 0, 0);
+
+	return 0;
 }
 
 
@@ -521,7 +353,7 @@ struct _xattr_key {
 };
 
 
-void
+int
 __gather_xattr_keys (dict_t *dict, char *key, data_t *value,
                      void *data)
 {
@@ -533,18 +365,19 @@ __gather_xattr_keys (dict_t *dict, char *key, data_t *value,
 
                 xkey = GF_CALLOC (1, sizeof (*xkey), gf_afr_mt_xattr_key);
                 if (!xkey)
-                        return;
+                        return -1;
 
                 xkey->key = key;
                 INIT_LIST_HEAD (&xkey->list);
 
                 list_add_tail (&xkey->list, list);
         }
+        return 0;
 }
 
 
 void
-__filter_xattrs (dict_t *dict)
+afr_filter_xattrs (dict_t *dict)
 {
         struct list_head   keys = {0,};
         struct _xattr_key *key  = NULL;
@@ -565,59 +398,56 @@ __filter_xattrs (dict_t *dict)
 }
 
 
-
-int32_t
+int
 afr_getxattr_cbk (call_frame_t *frame, void *cookie,
                   xlator_t *this, int32_t op_ret, int32_t op_errno,
                   dict_t *dict, dict_t *xdata)
 {
-        afr_private_t * priv            = NULL;
-        afr_local_t *   local           = NULL;
-        xlator_t **     children        = NULL;
-        int             unwind          = 1;
-        int32_t         *last_index     = NULL;
-        int32_t         next_call_child = -1;
-        int32_t         read_child      = -1;
-        int32_t         *fresh_children  = NULL;
-
-        priv     = this->private;
-        children = priv->children;
+        afr_local_t *local = NULL;
 
         local = frame->local;
 
-        read_child = (long) cookie;
+	if (op_ret < 0) {
+		local->op_ret = op_ret;
+		local->op_errno = op_errno;
 
-        if (op_ret == -1) {
-                last_index = &local->cont.getxattr.last_index;
-                fresh_children = local->fresh_children;
-                next_call_child = afr_next_call_child (fresh_children,
-                                                       local->child_up,
-                                                       priv->child_count,
-                                                       last_index, read_child);
-                if (next_call_child < 0)
-                        goto out;
-
-                unwind = 0;
-                STACK_WIND_COOKIE (frame, afr_getxattr_cbk,
-                                   (void *) (long) read_child,
-                                   children[next_call_child],
-                                   children[next_call_child]->fops->getxattr,
-                                   &local->loc,
-                                   local->cont.getxattr.name,
-                                   NULL);
-        }
+		afr_read_txn_continue (frame, this, (long) cookie);
+		return 0;
+	}
 
-out:
-        if (unwind) {
-                if (op_ret >= 0 && dict)
-                        __filter_xattrs (dict);
+	if (dict)
+		afr_filter_xattrs (dict);
 
-                AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, xdata);
-        }
+	AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, xdata);
 
         return 0;
 }
 
+
+int
+afr_getxattr_wind (call_frame_t *frame, xlator_t *this, int subvol)
+{
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+
+	local = frame->local;
+	priv = this->private;
+
+	if (subvol == -1) {
+                AFR_STACK_UNWIND (getxattr, frame, local->op_ret,
+				  local->op_errno, NULL, NULL);
+		return 0;
+	}
+
+	STACK_WIND_COOKIE (frame, afr_getxattr_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->getxattr,
+			   &local->loc, local->cont.getxattr.name,
+			   local->xdata_req);
+	return 0;
+}
+
+
 int32_t
 afr_getxattr_unwind (call_frame_t *frame, int op_ret, int op_errno,
                      dict_t *dict, dict_t *xdata)
@@ -628,6 +458,94 @@ afr_getxattr_unwind (call_frame_t *frame, int op_ret, int op_errno,
 }
 
 int32_t
+afr_fgetxattr_clrlk_cbk (call_frame_t *frame, void *cookie,
+                         xlator_t *this, int32_t op_ret, int32_t op_errno,
+                         dict_t *dict, dict_t *xdata)
+{
+        afr_local_t     *local                  = NULL;
+        afr_private_t   *priv                   = NULL;
+        xlator_t        **children              = NULL;
+        dict_t          *xattr                  = NULL;
+        char            *tmp_report             = NULL;
+        char            lk_summary[1024]        = {0,};
+        int             serz_len                = 0;
+        int32_t         callcnt                 = 0;
+        long int        cky                     = 0;
+        int             ret                     = 0;
+
+        priv     = this->private;
+        children = priv->children;
+
+        local = frame->local;
+        cky = (long) cookie;
+
+        LOCK (&frame->lock);
+        {
+                callcnt = --local->call_count;
+                if (op_ret == -1)
+                        local->replies[cky].op_errno = op_errno;
+
+                if (!local->dict)
+                        local->dict = dict_new ();
+                if (local->dict) {
+                        ret = dict_get_str (dict, local->cont.getxattr.name,
+                                            &tmp_report);
+                        if (ret)
+                                goto unlock;
+                        ret = dict_set_dynstr (local->dict,
+                                               children[cky]->name,
+                                               gf_strdup (tmp_report));
+                        if (ret)
+                                goto unlock;
+                }
+        }
+unlock:
+        UNLOCK (&frame->lock);
+
+        if (!callcnt) {
+                xattr = dict_new ();
+                if (!xattr) {
+                        op_ret = -1;
+                        op_errno = ENOMEM;
+                        goto unwind;
+                }
+                ret = dict_serialize_value_with_delim (local->dict,
+                                                       lk_summary,
+                                                       &serz_len, '\n');
+                if (ret) {
+                        op_ret = -1;
+                        op_errno = ENOMEM;
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Error serializing dictionary");
+                        goto unwind;
+                }
+                if (serz_len == -1)
+                        snprintf (lk_summary, sizeof (lk_summary),
+                                  "No locks cleared.");
+                ret = dict_set_dynstr (xattr, local->cont.getxattr.name,
+                                       gf_strdup (lk_summary));
+                if (ret) {
+                        op_ret = -1;
+                        op_errno = ENOMEM;
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Error setting dictionary");
+                        goto unwind;
+                }
+
+        unwind:
+                // Updating child_errno with more recent 'events'
+                op_errno = afr_final_errno (local, priv);
+
+                AFR_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, xattr,
+                                  xdata);
+                if (xattr)
+                        dict_unref (xattr);
+        }
+
+        return ret;
+}
+
+int32_t
 afr_getxattr_clrlk_cbk (call_frame_t *frame, void *cookie,
                         xlator_t *this, int32_t op_ret, int32_t op_errno,
                         dict_t *dict, dict_t *xdata)
@@ -653,7 +571,7 @@ afr_getxattr_clrlk_cbk (call_frame_t *frame, void *cookie,
         {
                 callcnt = --local->call_count;
                 if (op_ret == -1)
-                        local->child_errno[cky] = op_errno;
+                        local->replies[cky].op_errno = op_errno;
 
                 if (!local->dict)
                         local->dict = dict_new ();
@@ -704,9 +622,8 @@ unlock:
 
         unwind:
                 // Updating child_errno with more recent 'events'
-                local->child_errno[cky] = op_errno;
-                op_errno = afr_resultant_errno_get (NULL, local->child_errno,
-                                                    priv->child_count);
+                op_errno = afr_final_errno (local, priv);
+
                 AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, xattr, xdata);
 
                 if (xattr)
@@ -761,12 +678,372 @@ afr_getxattr_node_uuid_cbk (call_frame_t *frame, void *cookie,
 
  unwind:
         if (unwind)
-                AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, NULL);
+                AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict,
+                                  NULL);
+
+        return 0;
+}
+
+int32_t
+afr_getxattr_lockinfo_cbk (call_frame_t *frame, void *cookie,
+                           xlator_t *this, int32_t op_ret, int32_t op_errno,
+                           dict_t *dict, dict_t *xdata)
+{
+        int          call_cnt     = 0, len = 0;
+        char        *lockinfo_buf = NULL;
+        dict_t      *lockinfo     = NULL, *newdict = NULL;
+        afr_local_t *local        = NULL;
+
+        LOCK (&frame->lock);
+        {
+                local = frame->local;
+
+                call_cnt = --local->call_count;
+
+                if ((op_ret < 0) || (!dict && !xdata)) {
+                        goto unlock;
+                }
+
+                if (xdata) {
+                        if (!local->xdata_rsp) {
+                                local->xdata_rsp = dict_new ();
+                                if (!local->xdata_rsp) {
+                                        local->op_ret = -1;
+                                        local->op_errno = ENOMEM;
+                                        goto unlock;
+                                }
+                        }
+                }
+
+                if (!dict) {
+                        goto unlock;
+                }
+
+                op_ret = dict_get_ptr_and_len (dict, GF_XATTR_LOCKINFO_KEY,
+                                               (void **)&lockinfo_buf, &len);
+
+                if (!lockinfo_buf) {
+                        goto unlock;
+                }
+
+                if (!local->dict) {
+                        local->dict = dict_new ();
+                        if (!local->dict) {
+                                local->op_ret = -1;
+                                local->op_errno = ENOMEM;
+                                goto unlock;
+                        }
+                }
+        }
+unlock:
+        UNLOCK (&frame->lock);
+
+        if (lockinfo_buf != NULL) {
+                lockinfo = dict_new ();
+                if (lockinfo == NULL) {
+                        local->op_ret = -1;
+                        local->op_errno = ENOMEM;
+                } else {
+                        op_ret = dict_unserialize (lockinfo_buf, len,
+                                                   &lockinfo);
+
+                        if (lockinfo && local->dict) {
+                                dict_copy (lockinfo, local->dict);
+                        }
+                }
+        }
+
+        if (xdata && local->xdata_rsp) {
+                dict_copy (xdata, local->xdata_rsp);
+        }
+
+        if (!call_cnt) {
+                newdict = dict_new ();
+                if (!newdict) {
+                        local->op_ret = -1;
+                        local->op_errno = ENOMEM;
+                        goto unwind;
+                }
+
+                len = dict_serialized_length (local->dict);
+                if (len == 0) {
+                        goto unwind;
+                }
+
+                lockinfo_buf = GF_CALLOC (1, len, gf_common_mt_char);
+                if (!lockinfo_buf) {
+                        local->op_ret = -1;
+                        local->op_errno = ENOMEM;
+                        goto unwind;
+                }
+
+                op_ret = dict_serialize (local->dict, lockinfo_buf);
+                if (op_ret < 0) {
+                        local->op_ret = -1;
+                        local->op_errno = -op_ret;
+                }
+
+                op_ret = dict_set_dynptr (newdict, GF_XATTR_LOCKINFO_KEY,
+                                          (void *)lockinfo_buf, len);
+                if (op_ret < 0) {
+                        local->op_ret = -1;
+                        local->op_errno = -op_ret;
+                        goto unwind;
+                }
+
+        unwind:
+                AFR_STACK_UNWIND (getxattr, frame, op_ret,
+                                  op_errno, newdict,
+                                  local->xdata_rsp);
+        }
+
+        dict_unref (lockinfo);
+
+        return 0;
+}
+
+int32_t
+afr_fgetxattr_lockinfo_cbk (call_frame_t *frame, void *cookie,
+                            xlator_t *this, int32_t op_ret, int32_t op_errno,
+                            dict_t *dict, dict_t *xdata)
+{
+        int          call_cnt     = 0, len = 0;
+        char        *lockinfo_buf = NULL;
+        dict_t      *lockinfo     = NULL, *newdict = NULL;
+        afr_local_t *local        = NULL;
+
+        LOCK (&frame->lock);
+        {
+                local = frame->local;
+
+                call_cnt = --local->call_count;
+
+                if ((op_ret < 0) || (!dict && !xdata)) {
+                        goto unlock;
+                }
+
+                if (xdata) {
+                        if (!local->xdata_rsp) {
+                                local->xdata_rsp = dict_new ();
+                                if (!local->xdata_rsp) {
+                                        local->op_ret = -1;
+                                        local->op_errno = ENOMEM;
+                                        goto unlock;
+                                }
+                        }
+                }
+
+                if (!dict) {
+                        goto unlock;
+                }
+
+                op_ret = dict_get_ptr_and_len (dict, GF_XATTR_LOCKINFO_KEY,
+                                               (void **)&lockinfo_buf, &len);
+
+                if (!lockinfo_buf) {
+                        goto unlock;
+                }
+
+                if (!local->dict) {
+                        local->dict = dict_new ();
+                        if (!local->dict) {
+                                local->op_ret = -1;
+                                local->op_errno = ENOMEM;
+                                goto unlock;
+                        }
+                }
+        }
+unlock:
+        UNLOCK (&frame->lock);
+
+        if (lockinfo_buf != NULL) {
+                lockinfo = dict_new ();
+                if (lockinfo == NULL) {
+                        local->op_ret = -1;
+                        local->op_errno = ENOMEM;
+                } else {
+                        op_ret = dict_unserialize (lockinfo_buf, len,
+                                                   &lockinfo);
+
+                        if (lockinfo && local->dict) {
+                                dict_copy (lockinfo, local->dict);
+                        }
+                }
+        }
+
+        if (xdata && local->xdata_rsp) {
+                dict_copy (xdata, local->xdata_rsp);
+        }
+
+        if (!call_cnt) {
+                newdict = dict_new ();
+                if (!newdict) {
+                        local->op_ret = -1;
+                        local->op_errno = ENOMEM;
+                        goto unwind;
+                }
+
+                len = dict_serialized_length (local->dict);
+                if (len <= 0) {
+                        goto unwind;
+                }
+
+                lockinfo_buf = GF_CALLOC (1, len, gf_common_mt_char);
+                if (!lockinfo_buf) {
+                        local->op_ret = -1;
+                        local->op_errno = ENOMEM;
+                        goto unwind;
+                }
+
+                op_ret = dict_serialize (local->dict, lockinfo_buf);
+                if (op_ret < 0) {
+                        local->op_ret = -1;
+                        local->op_errno = -op_ret;
+                }
+
+                op_ret = dict_set_dynptr (newdict, GF_XATTR_LOCKINFO_KEY,
+                                          (void *)lockinfo_buf, len);
+                if (op_ret < 0) {
+                        local->op_ret = -1;
+                        local->op_errno = -op_ret;
+                        goto unwind;
+                }
+
+        unwind:
+                AFR_STACK_UNWIND (fgetxattr, frame, op_ret,
+                                  op_errno, newdict,
+                                  local->xdata_rsp);
+        }
+
+        dict_unref (lockinfo);
 
         return 0;
 }
 
 int32_t
+afr_fgetxattr_pathinfo_cbk (call_frame_t *frame, void *cookie,
+                            xlator_t *this, int32_t op_ret, int32_t op_errno,
+                            dict_t *dict, dict_t *xdata)
+{
+        afr_local_t *local          = NULL;
+        int32_t      callcnt        = 0;
+        int          ret            = 0;
+        char        *xattr          = NULL;
+        char        *xattr_serz     = NULL;
+        char        xattr_cky[1024] = {0,};
+        dict_t      *nxattr         = NULL;
+        long         cky            = 0;
+        int32_t      padding        = 0;
+        int32_t      tlen           = 0;
+
+        if (!frame || !frame->local || !this) {
+                gf_log ("", GF_LOG_ERROR, "possible NULL deref");
+                goto out;
+        }
+
+        local = frame->local;
+        cky = (long) cookie;
+
+        LOCK (&frame->lock);
+        {
+                callcnt = --local->call_count;
+
+                if (op_ret < 0) {
+                        local->op_errno = op_errno;
+                } else {
+                        local->op_ret = op_ret;
+                        if (!local->xdata_rsp && xdata)
+                                local->xdata_rsp = dict_ref (xdata);
+                }
+
+                if (!dict || (op_ret < 0))
+                        goto out;
+
+                if (!local->dict)
+                        local->dict = dict_new ();
+
+                if (local->dict) {
+                        ret = dict_get_str (dict,
+                                            local->cont.getxattr.name,
+                                            &xattr);
+                        if (ret)
+                                goto out;
+
+                        xattr = gf_strdup (xattr);
+
+                        (void)snprintf (xattr_cky, 1024, "%s-%ld",
+                                        local->cont.getxattr.name, cky);
+                        ret = dict_set_dynstr (local->dict,
+                                               xattr_cky, xattr);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Cannot set xattr cookie key");
+                                goto out;
+                        }
+
+                        local->cont.getxattr.xattr_len
+                                += strlen (xattr) + 1;
+                }
+        }
+out:
+        UNLOCK (&frame->lock);
+
+        if (!callcnt) {
+                if (!local->cont.getxattr.xattr_len)
+                        goto unwind;
+
+                nxattr = dict_new ();
+                if (!nxattr)
+                        goto unwind;
+
+                /* extra bytes for decorations (brackets and <>'s) */
+                padding += strlen (this->name)
+                        + strlen (AFR_PATHINFO_HEADER) + 4;
+                local->cont.getxattr.xattr_len += (padding + 2);
+
+                xattr_serz = GF_CALLOC (local->cont.getxattr.xattr_len,
+                                        sizeof (char), gf_common_mt_char);
+
+                if (!xattr_serz)
+                        goto unwind;
+
+                /* the xlator info */
+                (void) sprintf (xattr_serz, "(<"AFR_PATHINFO_HEADER"%s> ",
+                                this->name);
+
+                /* actual series of pathinfo */
+                ret = dict_serialize_value_with_delim (local->dict,
+                                                       xattr_serz
+                                                       + strlen (xattr_serz),
+                                                       &tlen, ' ');
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Error serializing"
+                                " dictionary");
+                        goto unwind;
+                }
+
+                /* closing part */
+                *(xattr_serz + padding + tlen) = ')';
+                *(xattr_serz + padding + tlen + 1) = '\0';
+
+                ret = dict_set_dynstr (nxattr, local->cont.getxattr.name,
+                                       xattr_serz);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR, "Cannot set pathinfo"
+                                " key in dict");
+
+        unwind:
+                AFR_STACK_UNWIND (fgetxattr, frame, local->op_ret,
+                                  local->op_errno, nxattr, local->xdata_rsp);
+
+                if (nxattr)
+                        dict_unref (nxattr);
+        }
+
+        return ret;
+}
+
+int32_t
 afr_getxattr_pathinfo_cbk (call_frame_t *frame, void *cookie,
                            xlator_t *this, int32_t op_ret, int32_t op_errno,
                            dict_t *dict, dict_t *xdata)
@@ -794,6 +1071,14 @@ afr_getxattr_pathinfo_cbk (call_frame_t *frame, void *cookie,
                 {
                         callcnt = --local->call_count;
 
+                        if (op_ret < 0) {
+                                local->op_errno = op_errno;
+                        } else {
+                                local->op_ret = op_ret;
+                                if (!local->xdata_rsp && xdata)
+                                        local->xdata_rsp = dict_ref (xdata);
+                        }
+
                         if (!dict || (op_ret < 0))
                                 goto out;
 
@@ -868,8 +1153,8 @@ afr_getxattr_pathinfo_cbk (call_frame_t *frame, void *cookie,
                                 " key in dict");
 
         unwind:
-                AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, nxattr,
-                                  xdata);
+                AFR_STACK_UNWIND (getxattr, frame, local->op_ret,
+                                  local->op_errno, nxattr, local->xdata_rsp);
 
                 if (nxattr)
                         dict_unref (nxattr);
@@ -878,51 +1163,133 @@ afr_getxattr_pathinfo_cbk (call_frame_t *frame, void *cookie,
         return ret;
 }
 
+static int
+afr_aggregate_stime_xattr (dict_t *this, char *key, data_t *value, void *data)
+{
+        int ret = 0;
+
+        if (fnmatch (GF_XATTR_STIME_PATTERN, key, FNM_NOESCAPE) == 0)
+                ret = gf_get_max_stime (THIS, data, key, value);
+
+        return ret;
+}
+
+int32_t
+afr_common_getxattr_stime_cbk (call_frame_t *frame, void *cookie,
+                               xlator_t *this, int32_t op_ret, int32_t op_errno,
+                               dict_t *dict, dict_t *xdata)
+{
+        afr_local_t *local          = NULL;
+        int32_t      callcnt        = 0;
+
+        if (!frame || !frame->local || !this) {
+                gf_log ("", GF_LOG_ERROR, "possible NULL deref");
+                goto out;
+        }
+
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                callcnt = --local->call_count;
+
+                if (!dict || (op_ret < 0)) {
+                        local->op_errno = op_errno;
+                        goto cleanup;
+                }
+
+                if (!local->dict)
+                        local->dict = dict_copy_with_ref (dict, NULL);
+                else
+                        dict_foreach (dict, afr_aggregate_stime_xattr,
+                                      local->dict);
+                local->op_ret = 0;
+        }
+
+cleanup:
+        UNLOCK (&frame->lock);
+
+        if (!callcnt) {
+                AFR_STACK_UNWIND (getxattr, frame, local->op_ret,
+                                  local->op_errno, local->dict, xdata);
+        }
+
+out:
+        return 0;
+}
+
+
 static gf_boolean_t
-afr_is_special_xattr (const char *name, fop_getxattr_cbk_t *cbk)
+afr_is_special_xattr (const char *name, fop_getxattr_cbk_t *cbk,
+                      gf_boolean_t is_fgetxattr)
 {
         gf_boolean_t    is_spl = _gf_true;
 
         GF_ASSERT (cbk);
-        if (!cbk) {
+        if (!cbk || !name) {
                 is_spl = _gf_false;
                 goto out;
         }
 
-        if (!strcmp (name, GF_XATTR_PATHINFO_KEY))
-                *cbk = afr_getxattr_pathinfo_cbk;
-
-        else if (!strncmp (name, GF_XATTR_CLRLK_CMD,
-                           strlen (GF_XATTR_CLRLK_CMD)))
-                *cbk = afr_getxattr_clrlk_cbk;
-        else
+        if (!strcmp (name, GF_XATTR_PATHINFO_KEY) ||
+	    !strcmp (name, GF_XATTR_USER_PATHINFO_KEY)) {
+                if (is_fgetxattr) {
+                        *cbk = afr_fgetxattr_pathinfo_cbk;
+                } else {
+                        *cbk = afr_getxattr_pathinfo_cbk;
+                }
+        } else if (!strncmp (name, GF_XATTR_CLRLK_CMD,
+                             strlen (GF_XATTR_CLRLK_CMD))) {
+                if (is_fgetxattr) {
+                        *cbk = afr_fgetxattr_clrlk_cbk;
+                } else {
+                        *cbk = afr_getxattr_clrlk_cbk;
+                }
+        } else if (!strncmp (name, GF_XATTR_LOCKINFO_KEY,
+                             strlen (GF_XATTR_LOCKINFO_KEY))) {
+                if (is_fgetxattr) {
+                        *cbk = afr_fgetxattr_lockinfo_cbk;
+                } else {
+                        *cbk = afr_getxattr_lockinfo_cbk;
+                }
+        } else if (fnmatch (GF_XATTR_STIME_PATTERN, name, FNM_NOESCAPE) == 0) {
+                *cbk = afr_common_getxattr_stime_cbk;
+        } else {
                 is_spl = _gf_false;
+        }
 
 out:
         return is_spl;
 }
 
 static void
-afr_getxattr_frm_all_children (xlator_t *this, call_frame_t *frame,
-                               const char *name, loc_t *loc,
-                               fop_getxattr_cbk_t cbk)
+afr_getxattr_all_subvols (xlator_t *this, call_frame_t *frame,
+			  const char *name, loc_t *loc,
+			  fop_getxattr_cbk_t cbk)
 {
         afr_private_t   *priv           = NULL;
         afr_local_t     *local          = NULL;
-        xlator_t        **children      = NULL;
         int             i               = 0;
+        int             call_count      = 0;
 
         priv     = this->private;
-        children = priv->children;
 
         local = frame->local;
-        local->call_count = priv->child_count;
+        //local->call_count set in afr_local_init
+        call_count = local->call_count;
+
+        //If up-children count is 0, afr_local_init would have failed already
+        //and the call would have unwound so not handling it here.
 
         for (i = 0; i < priv->child_count; i++) {
-                STACK_WIND_COOKIE (frame, cbk,
-                                   (void *) (long) i,
-                                   children[i], children[i]->fops->getxattr,
-                                   loc, name, NULL);
+                if (local->child_up[i]) {
+                        STACK_WIND_COOKIE (frame, cbk,
+                                           (void *) (long) i, priv->children[i],
+                                           priv->children[i]->fops->getxattr,
+                                           loc, name, NULL);
+                        if (!--call_count)
+                                break;
+                }
         }
         return;
 }
@@ -933,39 +1300,41 @@ afr_getxattr (call_frame_t *frame, xlator_t *this,
 {
         afr_private_t           *priv         = NULL;
         xlator_t                **children    = NULL;
-        int                     call_child    = 0;
         afr_local_t             *local        = NULL;
         xlator_list_t           *trav         = NULL;
         xlator_t                **sub_volumes = NULL;
         int                     i             = 0;
         int32_t                 op_errno      = 0;
-        int32_t                 read_child    = -1;
         int                     ret           = -1;
         fop_getxattr_cbk_t      cbk           = NULL;
+        int                     afr_xtime_gauge[MCNT_MAX] = {0,};
 
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
 
         priv     = this->private;
-        VALIDATE_OR_GOTO (priv->children, out);
 
         children = priv->children;
 
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
+        loc_copy (&local->loc, loc);
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+	local->op = GF_FOP_GETXATTR;
+
+	if (xdata)
+		local->xdata_req = dict_ref (xdata);
 
-        loc_copy (&local->loc, loc);
         if (!name)
                 goto no_name;
 
         local->cont.getxattr.name = gf_strdup (name);
 
+	if (!local->cont.getxattr.name) {
+		op_errno = ENOMEM;
+		goto out;
+	}
+
         if (!strncmp (name, AFR_XATTR_PREFIX,
                       strlen (AFR_XATTR_PREFIX))) {
                 gf_log (this->name, GF_LOG_INFO,
@@ -975,7 +1344,7 @@ afr_getxattr (call_frame_t *frame, xlator_t *this,
                 goto out;
         }
         if ((strcmp (GF_XATTR_MARKER_KEY, name) == 0)
-            && (-1 == frame->root->pid)) {
+            && (GF_CLIENT_PID_GSYNCD == frame->root->pid)) {
 
                 local->marker.call_count = priv->child_count;
 
@@ -991,6 +1360,7 @@ afr_getxattr (call_frame_t *frame, xlator_t *this,
                                            sub_volumes,
                                            priv->child_count,
                                            MARKER_UUID_TYPE,
+                                           marker_uuid_default_gauge,
                                            priv->vol_uuid)) {
 
                         gf_log (this->name, GF_LOG_INFO,
@@ -1007,9 +1377,8 @@ afr_getxattr (call_frame_t *frame, xlator_t *this,
          * if we are doing getxattr with pathinfo as the key then we
          * collect information from all childs
          */
-        if (afr_is_special_xattr (name, &cbk)) {
-                afr_getxattr_frm_all_children (this, frame, name,
-                                               loc, cbk);
+        if (afr_is_special_xattr (name, &cbk, 0)) {
+                afr_getxattr_all_subvols (this, frame, name, loc, cbk);
                 return 0;
         }
 
@@ -1025,10 +1394,11 @@ afr_getxattr (call_frame_t *frame, xlator_t *this,
 
         if (*priv->vol_uuid) {
                 if ((match_uuid_local (name, priv->vol_uuid) == 0)
-                    && (-1 == frame->root->pid)) {
+                    && (GF_CLIENT_PID_GSYNCD == frame->root->pid)) {
                         local->marker.call_count = priv->child_count;
 
-                        sub_volumes = alloca ( priv->child_count * sizeof (xlator_t *));
+                        sub_volumes = alloca ( priv->child_count
+                                               * sizeof (xlator_t *));
                         for (i = 0, trav = this->children; trav ;
                              trav = trav->next, i++) {
 
@@ -1036,12 +1406,20 @@ afr_getxattr (call_frame_t *frame, xlator_t *this,
 
                         }
 
+                        /* don't err out on getting ENOTCONN (brick down)
+                         * from a subset of the bricks
+                         */
+                        memcpy (afr_xtime_gauge, marker_xtime_default_gauge,
+                                sizeof (afr_xtime_gauge));
+                        afr_xtime_gauge[MCNT_NOTFOUND] = 0;
+                        afr_xtime_gauge[MCNT_ENOTCONN] = 0;
                         if (cluster_getmarkerattr (frame, this, loc,
                                                    name, local,
                                                    afr_getxattr_unwind,
                                                    sub_volumes,
                                                    priv->child_count,
                                                    MARKER_XTIME_TYPE,
+                                                   afr_xtime_gauge,
                                                    priv->vol_uuid)) {
                                 gf_log (this->name, GF_LOG_INFO,
                                         "%s: failed to get marker attr (%s)",
@@ -1055,27 +1433,9 @@ afr_getxattr (call_frame_t *frame, xlator_t *this,
         }
 
 no_name:
-        local->fresh_children = afr_children_create (priv->child_count);
-        if (!local->fresh_children) {
-                op_errno = ENOMEM;
-                goto out;
-        }
 
-        read_child = afr_inode_get_read_ctx (this, loc->inode, local->fresh_children);
-        ret = afr_get_call_child (this, local->child_up, read_child,
-                                     local->fresh_children,
-                                     &call_child,
-                                     &local->cont.getxattr.last_index);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
-        }
-
-        STACK_WIND_COOKIE (frame, afr_getxattr_cbk,
-                           (void *) (long) call_child,
-                           children[call_child],
-                           children[call_child]->fops->getxattr,
-                           loc, name, xdata);
+	afr_read_txn (frame, this, local->loc.inode, afr_getxattr_wind,
+		      AFR_METADATA_TRANSACTION);
 
         ret = 0;
 out:
@@ -1092,128 +1452,128 @@ afr_fgetxattr_cbk (call_frame_t *frame, void *cookie,
                    xlator_t *this, int32_t op_ret, int32_t op_errno,
                    dict_t *dict, dict_t *xdata)
 {
-        afr_private_t * priv            = NULL;
-        afr_local_t *   local           = NULL;
-        xlator_t **     children        = NULL;
-        int             unwind          = 1;
-        int32_t         *last_index     = NULL;
-        int32_t         next_call_child = -1;
-        int32_t         read_child      = -1;
-        int32_t         *fresh_children  = NULL;
+	afr_local_t *local = NULL;
 
-        priv     = this->private;
-        children = priv->children;
+	local = frame->local;
 
-        local = frame->local;
+	if (op_ret < 0) {
+		local->op_ret = -1;
+		local->op_errno = op_errno;
 
-        read_child = (long) cookie;
+		afr_read_txn_continue (frame, this, (long) cookie);
+		return 0;
+	}
 
-        if (op_ret == -1) {
-                last_index = &local->cont.getxattr.last_index;
-                fresh_children = local->fresh_children;
-                next_call_child = afr_next_call_child (fresh_children,
-                                                       local->child_up,
-                                                       priv->child_count,
-                                                       last_index, read_child);
-                if (next_call_child < 0)
-                        goto out;
+	if (dict)
+		afr_filter_xattrs (dict);
 
-                unwind = 0;
-                STACK_WIND_COOKIE (frame, afr_fgetxattr_cbk,
-                                   (void *) (long) read_child,
-                                   children[next_call_child],
-                                   children[next_call_child]->fops->fgetxattr,
-                                   local->fd,
-                                   local->cont.getxattr.name,
-                                   NULL);
-        }
+	AFR_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, dict, xdata);
 
-out:
-        if (unwind) {
-                if (op_ret >= 0 && dict)
-                        __filter_xattrs (dict);
-
-                AFR_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, dict,
-                                  xdata);
-        }
-
-        return 0;
+	return 0;
 }
 
-int32_t
-afr_fgetxattr_unwind (call_frame_t *frame,
-                      int op_ret, int op_errno, dict_t *dict, dict_t *xdata)
-
+int
+afr_fgetxattr_wind (call_frame_t *frame, xlator_t *this, int subvol)
 {
-        AFR_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, dict, xdata);
-        return 0;
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+
+	local = frame->local;
+	priv = this->private;
+
+	if (subvol == -1) {
+		AFR_STACK_UNWIND (fgetxattr, frame, local->op_ret,
+				  local->op_errno, NULL, NULL);
+		return 0;
+	}
+
+	STACK_WIND_COOKIE (frame, (void *) (long) subvol, afr_fgetxattr_cbk,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->fgetxattr,
+			   local->fd, local->cont.getxattr.name,
+			   local->xdata_req);
+	return 0;
 }
 
-int32_t
-afr_fgetxattr (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, const char *name, dict_t *xdata)
-{
-        afr_private_t   *priv         = NULL;
-        xlator_t        **children    = NULL;
-        int             call_child    = 0;
-        afr_local_t     *local        = NULL;
-        int32_t         op_ret        = -1;
-        int32_t         op_errno      = 0;
-        int32_t         read_child    = -1;
-
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+static void
+afr_fgetxattr_all_subvols (xlator_t *this, call_frame_t *frame,
+			   fop_fgetxattr_cbk_t cbk)
+{
+        afr_private_t   *priv           = NULL;
+        afr_local_t     *local          = NULL;
+        int             i               = 0;
+        int             call_count      = 0;
 
         priv     = this->private;
-        VALIDATE_OR_GOTO (priv->children, out);
 
-        children = priv->children;
+        local = frame->local;
+        //local->call_count set in afr_local_init
+        call_count = local->call_count;
 
-        AFR_LOCAL_ALLOC_OR_GOTO (local, out);
-        frame->local = local;
+        //If up-children count is 0, afr_local_init would have failed already
+        //and the call would have unwound so not handling it here.
 
-        op_ret = afr_local_init (local, priv, &op_errno);
-        if (op_ret < 0) {
-                op_errno = -op_ret;
-                goto out;
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->child_up[i]) {
+                        STACK_WIND_COOKIE (frame, cbk,
+                                           (void *) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->fgetxattr,
+                                           local->fd, local->cont.getxattr.name,
+					   NULL);
+                        if (!--call_count)
+                                break;
+                }
         }
 
-        local->fd = fd_ref (fd);
-        if (name)
-                local->cont.getxattr.name = gf_strdup (name);
+        return;
+}
 
-        /* pathinfo gets handled only in getxattr() */
 
-        local->fresh_children = afr_children_create (priv->child_count);
-        if (!local->fresh_children) {
-                op_errno = ENOMEM;
-                goto out;
-        }
+int
+afr_fgetxattr (call_frame_t *frame, xlator_t *this,
+               fd_t *fd, const char *name, dict_t *xdata)
+{
+        afr_local_t          *local      = NULL;
+        int32_t               op_errno   = 0;
+        fop_fgetxattr_cbk_t   cbk        = NULL;
 
-        read_child = afr_inode_get_read_ctx (this, fd->inode, local->fresh_children);
-        op_ret = afr_get_call_child (this, local->child_up, read_child,
-                                     local->fresh_children,
-                                     &call_child,
-                                     &local->cont.getxattr.last_index);
-        if (op_ret < 0) {
-                op_errno = -op_ret;
-                op_ret = -1;
-                goto out;
+        local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
+
+	local->op = GF_FOP_FGETXATTR;
+        local->fd = fd_ref (fd);
+        if (name) {
+                local->cont.getxattr.name = gf_strdup (name);
+		if (!local->cont.getxattr.name) {
+			op_errno = ENOMEM;
+			goto out;
+		}
+	}
+	if (xdata)
+		local->xdata_req = dict_ref (xdata);
+
+        /* pathinfo gets handled only in getxattr(), but we need to handle
+         * lockinfo.
+         * If we are doing fgetxattr with lockinfo as the key then we
+         * collect information from all children.
+         */
+        if (afr_is_special_xattr (name, &cbk, 1)) {
+                afr_fgetxattr_all_subvols (this, frame, cbk);
+                return 0;
         }
 
-        STACK_WIND_COOKIE (frame, afr_fgetxattr_cbk,
-                           (void *) (long) call_child,
-                           children[call_child],
-                           children[call_child]->fops->fgetxattr,
-                           fd, name, xdata);
+        afr_fix_open (fd, this);
 
-        op_ret = 0;
+	afr_read_txn (frame, this, fd->inode, afr_fgetxattr_wind,
+		      AFR_METADATA_TRANSACTION);
+
+	return 0;
 out:
-        if (op_ret == -1) {
-                AFR_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, NULL, NULL);
-        }
+	AFR_STACK_UNWIND (fgetxattr, frame, -1, op_errno, NULL, NULL);
+
         return 0;
 }
 
@@ -1222,145 +1582,84 @@ out:
 
 /* {{{ readv */
 
-/**
- * read algorithm:
- *
- * if the user has specified a read subvolume, use it
- * otherwise -
- *   use the inode number to hash it to one of the subvolumes, and
- *   read from there (to balance read load)
- *
- * if any of the above read's fail, try the children in sequence
- * beginning at the beginning
- */
-
-int32_t
+int
 afr_readv_cbk (call_frame_t *frame, void *cookie,
                xlator_t *this, int32_t op_ret, int32_t op_errno,
                struct iovec *vector, int32_t count, struct iatt *buf,
                struct iobref *iobref, dict_t *xdata)
 {
-        afr_private_t * priv            = NULL;
-        afr_local_t *   local           = NULL;
-        xlator_t **     children        = NULL;
-        int             unwind          = 1;
-        int32_t         *last_index     = NULL;
-        int32_t         next_call_child = -1;
-        int32_t         *fresh_children  = NULL;
-        int32_t         read_child      = -1;
+	afr_local_t *local = NULL;
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+	local = frame->local;
 
-        priv     = this->private;
-        VALIDATE_OR_GOTO (priv->children, out);
+	if (op_ret < 0) {
+		local->op_ret = -1;
+		local->op_errno = op_errno;
 
-        children = priv->children;
-
-        local = frame->local;
-
-        read_child = (long) cookie;
-
-        if (op_ret == -1) {
-                last_index = &local->cont.readv.last_index;
-                fresh_children = local->fresh_children;
-                next_call_child = afr_next_call_child (fresh_children,
-                                                       local->child_up,
-                                                       priv->child_count,
-                                                       last_index, read_child);
-                if (next_call_child < 0)
-                        goto out;
-
-                unwind = 0;
+		afr_read_txn_continue (frame, this, (long) cookie);
+		return 0;
+	}
 
-                STACK_WIND_COOKIE (frame, afr_readv_cbk,
-                                   (void *) (long) read_child,
-                                   children[next_call_child],
-                                   children[next_call_child]->fops->readv,
-                                   local->fd, local->cont.readv.size,
-                                   local->cont.readv.offset,
-                                   local->cont.readv.flags,
-                                   NULL);
-        }
+	AFR_STACK_UNWIND (readv, frame, op_ret, op_errno,
+			  vector, count, buf, iobref, xdata);
+	return 0;
+}
 
-out:
-        if (unwind) {
-                AFR_STACK_UNWIND (readv, frame, op_ret, op_errno,
-                                  vector, count, buf, iobref, xdata);
-        }
 
-        return 0;
+int
+afr_readv_wind (call_frame_t *frame, xlator_t *this, int subvol)
+{
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+
+	local = frame->local;
+	priv = this->private;
+
+	if (subvol == -1) {
+		AFR_STACK_UNWIND (readv, frame, local->op_ret, local->op_errno,
+				  0, 0, 0, 0, 0);
+		return 0;
+	}
+
+	STACK_WIND_COOKIE (frame, afr_readv_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->readv,
+			   local->fd, local->cont.readv.size,
+			   local->cont.readv.offset, local->cont.readv.flags,
+			   local->xdata_req);
+	return 0;
 }
 
 
-int32_t
-afr_readv (call_frame_t *frame, xlator_t *this,
-           fd_t *fd, size_t size, off_t offset, uint32_t flags, dict_t *xdata)
+int
+afr_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+	   off_t offset, uint32_t flags, dict_t *xdata)
 {
-        afr_private_t * priv       = NULL;
         afr_local_t   * local      = NULL;
-        xlator_t **     children   = NULL;
-        int             call_child = 0;
         int32_t         op_errno   = 0;
-        int32_t         read_child = -1;
-        int             ret        = -1;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
-        VALIDATE_OR_GOTO (fd, out);
-
-        priv     = this->private;
-        children = priv->children;
-
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
 
-        local->fresh_children = afr_children_create (priv->child_count);
-        if (!local->fresh_children) {
-                op_errno = ENOMEM;
-                goto out;
-        }
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
 
-        read_child = afr_inode_get_read_ctx (this, fd->inode, local->fresh_children);
-        ret = afr_get_call_child (this, local->child_up, read_child,
-                                     local->fresh_children,
-                                     &call_child,
-                                     &local->cont.readv.last_index);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
-        }
+	local->op = GF_FOP_READ;
+        local->fd = fd_ref (fd);
+        local->cont.readv.size = size;
+        local->cont.readv.offset = offset;
+        local->cont.readv.flags = flags;
+	if (xdata)
+		local->xdata_req = dict_ref (xdata);
 
-        local->fd                    = fd_ref (fd);
+        afr_fix_open (fd, this);
 
-        local->cont.readv.size       = size;
-        local->cont.readv.offset     = offset;
-        local->cont.readv.flags      = flags;
+	afr_read_txn (frame, this, fd->inode, afr_readv_wind,
+		      AFR_DATA_TRANSACTION);
 
-        ret = afr_open_fd_fix (frame, this, _gf_false);
-        if (ret) {
-                op_errno = -ret;
-                goto out;
-        }
-        STACK_WIND_COOKIE (frame, afr_readv_cbk,
-                           (void *) (long) call_child,
-                           children[call_child],
-                           children[call_child]->fops->readv,
-                           fd, size, offset, flags, xdata);
-
-        ret = 0;
-out:
-        if (ret < 0) {
-                AFR_STACK_UNWIND (readv, frame, -1, op_errno, NULL, 0, NULL,
-                                  NULL, NULL);
-        }
         return 0;
+out:
+	AFR_STACK_UNWIND(readv, frame, -1, op_errno, 0, 0, 0, 0, 0);
+
+	return 0;
 }
 
 /* }}} */
diff --git a/xlators/cluster/afr/src/afr-inode-write.c b/xlators/cluster/afr/src/afr-inode-write.c
index 02e36b13a..3dacfc8dd 100644
--- a/xlators/cluster/afr/src/afr-inode-write.c
+++ b/xlators/cluster/afr/src/afr-inode-write.c
@@ -37,145 +37,303 @@
 
 #include "afr.h"
 #include "afr-transaction.h"
-#include "afr-self-heal-common.h"
+//#include "afr-self-heal-common.h"
 
-/* {{{ writev */
 
-int
-afr_writev_unwind (call_frame_t *frame, xlator_t *this)
+static void
+__afr_inode_write_finalize (call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	int read_subvol = 0;
+	int i = 0;
+
+	local = frame->local;
+	priv = this->private;
+
+	if (local->inode) {
+		if (local->transaction.type == AFR_METADATA_TRANSACTION)
+			read_subvol = afr_metadata_subvol_get (local->inode, this,
+							       NULL, NULL);
+		else
+			read_subvol = afr_data_subvol_get (local->inode, this,
+							   NULL, NULL);
+	}
+
+	local->op_ret = -1;
+	local->op_errno = afr_final_errno (local, priv);
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!local->replies[i].valid)
+			continue;
+		if (local->replies[i].op_ret < 0) {
+			afr_inode_read_subvol_reset (local->inode, this);
+			continue;
+		}
+
+		/* Order of checks in the compound conditional
+		   below is important.
+
+		   - Highest precedence: largest op_ret
+		   - Next precendence: if all op_rets are equal, read subvol
+		   - Least precedence: any succeeded subvol
+		*/
+		if ((local->op_ret < local->replies[i].op_ret) ||
+		    ((local->op_ret == local->replies[i].op_ret) &&
+		     (i == read_subvol))) {
+
+			local->op_ret = local->replies[i].op_ret;
+			local->op_errno = local->replies[i].op_errno;
+
+			local->cont.inode_wfop.prebuf =
+				local->replies[i].prestat;
+			local->cont.inode_wfop.postbuf =
+				local->replies[i].poststat;
+
+			if (local->replies[i].xdata) {
+				if (local->xdata_rsp)
+					dict_unref (local->xdata_rsp);
+				local->xdata_rsp =
+					dict_ref (local->replies[i].xdata);
+			}
+		}
+	}
+}
+
+
+static void
+__afr_inode_write_fill (call_frame_t *frame, xlator_t *this, int child_index,
+			int op_ret, int op_errno,
+			struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+{
+        afr_local_t *local = NULL;
+
+        local = frame->local;
+
+	local->replies[child_index].valid = 1;
+	local->replies[child_index].op_ret = op_ret;
+	local->replies[child_index].op_errno = op_errno;
+
+	if (op_ret >= 0) {
+		if (prebuf)
+			local->replies[child_index].prestat = *prebuf;
+		if (postbuf)
+			local->replies[child_index].poststat = *postbuf;
+		if (xdata)
+			local->replies[child_index].xdata = dict_ref (xdata);
+	} else {
+		afr_transaction_fop_failed (frame, this, child_index);
+	}
+
+        return;
+}
+
+
+static int
+__afr_inode_write_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                       struct iatt *postbuf, dict_t *xdata)
+{
+        afr_local_t *local = NULL;
+        int child_index = (long) cookie;
+        int call_count = -1;
 
         local = frame->local;
 
         LOCK (&frame->lock);
         {
-                if (local->transaction.main_frame)
-                        main_frame = local->transaction.main_frame;
-                local->transaction.main_frame = NULL;
+                __afr_inode_write_fill (frame, this, child_index, op_ret,
+					op_errno, prebuf, postbuf, xdata);
         }
         UNLOCK (&frame->lock);
 
-        if (main_frame) {
-                AFR_STACK_UNWIND (writev, main_frame,
-                                  local->op_ret, local->op_errno,
-                                  &local->cont.writev.prebuf,
-                                  &local->cont.writev.postbuf,
-                                  NULL);
+        call_count = afr_frame_return (frame);
+
+        if (call_count == 0) {
+		__afr_inode_write_finalize (frame, this);
+
+		if (afr_txn_nothing_failed (frame, this))
+			local->transaction.unwind (frame, this);
+
+                local->transaction.resume (frame, this);
         }
+
         return 0;
 }
 
+/* {{{ writev */
 
-int
-afr_writev_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                     struct iatt *postbuf, dict_t *xdata)
+void
+afr_writev_copy_outvars (call_frame_t *src_frame, call_frame_t *dst_frame)
 {
-        afr_local_t *   local = NULL;
-        int child_index = (long) cookie;
-        int call_count  = -1;
-        int read_child  = 0;
+        afr_local_t *src_local = NULL;
+        afr_local_t *dst_local = NULL;
+
+        src_local = src_frame->local;
+        dst_local = dst_frame->local;
+
+        dst_local->op_ret = src_local->op_ret;
+        dst_local->op_errno = src_local->op_errno;
+        dst_local->cont.inode_wfop.prebuf = src_local->cont.inode_wfop.prebuf;
+        dst_local->cont.inode_wfop.postbuf = src_local->cont.inode_wfop.postbuf;
+	if (src_local->xdata_rsp)
+		dst_local->xdata_rsp = dict_ref (src_local->xdata_rsp);
+}
 
+void
+afr_writev_unwind (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t *   local = NULL;
         local = frame->local;
 
-        read_child = afr_inode_get_read_ctx (this, local->fd->inode, NULL);
-
-        LOCK (&frame->lock);
-        {
-                if (child_index == read_child) {
-                        local->read_child_returned = _gf_true;
-                }
+        AFR_STACK_UNWIND (writev, frame,
+                          local->op_ret, local->op_errno,
+                          &local->cont.inode_wfop.prebuf,
+                          &local->cont.inode_wfop.postbuf,
+                          local->xdata_rsp);
+}
 
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
 
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret              = op_ret;
-                                local->cont.writev.prebuf  = *prebuf;
-                                local->cont.writev.postbuf = *postbuf;
-                        }
+int
+afr_transaction_writev_unwind (call_frame_t *frame, xlator_t *this)
+{
+        call_frame_t *fop_frame = NULL;
 
-                        if (child_index == read_child) {
-                                local->cont.writev.prebuf  = *prebuf;
-                                local->cont.writev.postbuf = *postbuf;
-                        }
-                }
+        fop_frame = afr_transaction_detach_fop_frame (frame);
 
-                local->op_errno = op_errno;
+        if (fop_frame) {
+                afr_writev_copy_outvars (frame, fop_frame);
+                afr_writev_unwind (fop_frame, this);
         }
-        UNLOCK (&frame->lock);
+        return 0;
+}
 
-        call_count = afr_frame_return (frame);
+static void
+afr_writev_handle_short_writes (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t   *local = NULL;
+        afr_private_t *priv  = NULL;
+        int           i      = 0;
 
-        if (call_count == 0) {
-                local->transaction.unwind (frame, this);
+        local = frame->local;
+        priv = this->private;
+        /*
+         * We already have the best case result of the writev calls staged
+         * as the return value. Any writev that returns some value less
+         * than the best case is now out of sync, so mark the fop as
+         * failed. Note that fops that have returned with errors have
+         * already been marked as failed.
+         */
+        for (i = 0; i < priv->child_count; i++) {
+                if ((!local->replies[i].valid) ||
+                    (local->replies[i].op_ret == -1))
+                        continue;
 
-                local->transaction.resume (frame, this);
+                if (local->replies[i].op_ret < local->op_ret)
+                        afr_transaction_fop_failed(frame, this, i);
         }
-        return 0;
 }
 
 int
-afr_writev_wind (call_frame_t *frame, xlator_t *this)
+afr_writev_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        int i = 0;
-        int call_count = -1;
+        afr_local_t *   local = NULL;
+        call_frame_t    *fop_frame = NULL;
+        int child_index = (long) cookie;
+        int call_count  = -1;
+        int ret = 0;
+        uint32_t open_fd_count = 0;
+        uint32_t write_is_append = 0;
 
         local = frame->local;
-        priv = this->private;
 
-        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
-                                                     priv->child_count);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
+        LOCK (&frame->lock);
+        {
+                __afr_inode_write_fill (frame, this, child_index, op_ret,
+					op_errno, prebuf, postbuf, xdata);
+		if (op_ret == -1 || !xdata)
+			goto unlock;
+
+		write_is_append = 0;
+		ret = dict_get_uint32 (xdata, GLUSTERFS_WRITE_IS_APPEND,
+				       &write_is_append);
+		if (ret || !write_is_append)
+			local->append_write = _gf_false;
+
+		ret = dict_get_uint32 (xdata, GLUSTERFS_OPEN_FD_COUNT,
+				       &open_fd_count);
+		if (ret == -1)
+			goto unlock;
+		if ((open_fd_count > local->open_fd_count)) {
+			local->open_fd_count = open_fd_count;
+			local->update_open_fd_count = _gf_true;
+		}
         }
+unlock:
+        UNLOCK (&frame->lock);
 
-        local->call_count = call_count;
+        call_count = afr_frame_return (frame);
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i]) {
-                        STACK_WIND_COOKIE (frame, afr_writev_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->writev,
-                                           local->fd,
-                                           local->cont.writev.vector,
-                                           local->cont.writev.count,
-                                           local->cont.writev.offset,
-                                           local->cont.writev.flags,
-                                           local->cont.writev.iobref,
-                                           NULL);
-
-                        if (!--call_count)
-                                break;
+        if (call_count == 0) {
+		if (!local->stable_write && !local->append_write)
+			/* An appended write removes the necessity to
+			   fsync() the file. This is because self-heal
+			   has the logic to check for larger file when
+			   the xattrs are not reliably pointing at
+			   a stale file.
+			*/
+			afr_fd_report_unstable_write (this, local->fd);
+
+		__afr_inode_write_finalize (frame, this);
+
+                afr_writev_handle_short_writes (frame, this);
+
+                if (local->update_open_fd_count)
+                        afr_handle_open_fd_count (frame, this);
+
+                if (!afr_txn_nothing_failed (frame, this)) {
+                        //Don't unwind until post-op is complete
+                        local->transaction.resume (frame, this);
+                } else {
+                /*
+                 * Generally inode-write fops do transaction.unwind then
+                 * transaction.resume, but writev needs to make sure that
+                 * delayed post-op frame is placed in fdctx before unwind
+                 * happens. This prevents the race of flush doing the
+                 * changelog wakeup first in fuse thread and then this
+                 * writev placing its delayed post-op frame in fdctx.
+                 * This helps flush make sure all the delayed post-ops are
+                 * completed.
+                 */
+
+                        fop_frame = afr_transaction_detach_fop_frame (frame);
+                        afr_writev_copy_outvars (frame, fop_frame);
+                        local->transaction.resume (frame, this);
+                        afr_writev_unwind (fop_frame, this);
                 }
         }
-
         return 0;
 }
 
 
 int
-afr_writev_done (call_frame_t *frame, xlator_t *this)
+afr_writev_wind (call_frame_t *frame, xlator_t *this, int subvol)
 {
         afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
 
         local = frame->local;
+        priv = this->private;
 
-        iobref_unref (local->cont.writev.iobref);
-        local->cont.writev.iobref = NULL;
-
-        local->transaction.unwind (frame, this);
-
-        AFR_STACK_DESTROY (frame);
-
+	STACK_WIND_COOKIE (frame, afr_writev_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->writev,
+			   local->fd, local->cont.writev.vector,
+			   local->cont.writev.count, local->cont.writev.offset,
+			   local->cont.writev.flags, local->cont.writev.iobref,
+			   local->xdata_req);
         return 0;
 }
 
@@ -185,30 +343,37 @@ afr_do_writev (call_frame_t *frame, xlator_t *this)
 {
         call_frame_t    *transaction_frame = NULL;
         afr_local_t     *local             = NULL;
-        int             op_ret   = -1;
-        int             op_errno = 0;
-
-        local = frame->local;
+        int             ret   = -1;
+        int             op_errno = ENOMEM;
 
         transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                op_errno = ENOMEM;
+        if (!transaction_frame)
                 goto out;
-        }
 
+        local = frame->local;
         transaction_frame->local = local;
-        frame->local = NULL;
+	frame->local = NULL;
 
-        local->op = GF_FOP_WRITE;
+	if (!AFR_FRAME_INIT (frame, op_errno))
+		goto out;
 
-        local->success_count      = 0;
+        local->op = GF_FOP_WRITE;
 
-        local->transaction.fop    = afr_writev_wind;
-        local->transaction.done   = afr_writev_done;
-        local->transaction.unwind = afr_writev_unwind;
+        local->transaction.wind   = afr_writev_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
+        local->transaction.unwind = afr_transaction_writev_unwind;
 
         local->transaction.main_frame = frame;
+
         if (local->fd->flags & O_APPEND) {
+               /*
+                * Backend vfs ignores the 'offset' for append mode fd so
+                * locking just the region provided for the writev does not
+                * give consistency gurantee. The actual write may happen at a
+                * completely different range than the one provided by the
+                * offset, len in the fop. So lock the entire file.
+                */
                 local->transaction.start   = 0;
                 local->transaction.len     = 0;
         } else {
@@ -217,269 +382,86 @@ afr_do_writev (call_frame_t *frame, xlator_t *this)
                                                          local->cont.writev.count);
         }
 
-        afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
-
-        op_ret = 0;
-out:
-        if (op_ret == -1) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (writev, frame, op_ret, op_errno, NULL, NULL, NULL);
-        }
-
-        return 0;
-}
-
-static int
-afr_prepare_loc (call_frame_t *frame, fd_t *fd)
-{
-        afr_local_t    *local = NULL;
-        char           *name = NULL;
-        char           *path = NULL;
-        int             ret = 0;
-
-        if ((!fd) || (!fd->inode))
-                return -1;
-
-        local = frame->local;
-        ret = inode_path (fd->inode, NULL, (char **)&path);
-        if (ret <= 0) {
-                gf_log (frame->this->name, GF_LOG_DEBUG,
-                        "Unable to get path for gfid: %s",
-                        uuid_utoa (fd->inode->gfid));
-                return -1;
-        }
-
-        if (local->loc.path) {
-                if (strcmp (path, local->loc.path))
-                        gf_log (frame->this->name, GF_LOG_DEBUG,
-                                "overwriting old loc->path %s with %s",
-                                local->loc.path, path);
-                GF_FREE ((char *)local->loc.path);
-        }
-        local->loc.path = path;
-
-        name = strrchr (local->loc.path, '/');
-        if (name)
-                name++;
-        local->loc.name = name;
-
-        if (local->loc.inode) {
-                inode_unref (local->loc.inode);
-        }
-        local->loc.inode = inode_ref (fd->inode);
-
-        if (local->loc.parent) {
-                inode_unref (local->loc.parent);
+        ret = afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
+        if (ret < 0) {
+            op_errno = -ret;
+            goto out;
         }
 
-        local->loc.parent = inode_parent (local->loc.inode, 0, NULL);
+	return 0;
+out:
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
 
+	AFR_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
-afr_fd_paused_call_t*
-afr_paused_call_create (call_frame_t *frame)
-{
-        afr_local_t             *local = NULL;
-        afr_fd_paused_call_t   *paused_call = NULL;
-
-        local = frame->local;
-        GF_ASSERT (local->fop_call_continue);
-
-        paused_call = GF_CALLOC (1, sizeof (*paused_call),
-                                  gf_afr_fd_paused_call_t);
-        if (paused_call) {
-                INIT_LIST_HEAD (&paused_call->call_list);
-                paused_call->frame = frame;
-        }
-
-        return paused_call;
-}
-
-static int
-afr_pause_fd_fop (call_frame_t *frame, xlator_t *this, afr_fd_ctx_t *fd_ctx)
-{
-        afr_fd_paused_call_t *paused_call = NULL;
-        int                    ret = 0;
-
-        paused_call = afr_paused_call_create (frame);
-        if (paused_call)
-                list_add (&paused_call->call_list, &fd_ctx->paused_calls);
-        else
-                ret = -ENOMEM;
-
-        return ret;
-}
-
-static void
-afr_trigger_open_fd_self_heal (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t             *local = NULL;
-        afr_self_heal_t         *sh = NULL;
-        inode_t                 *inode = NULL;
-        char                    *reason = NULL;
-
-        local = frame->local;
-        sh    = &local->self_heal;
-        inode = local->fd->inode;
-
-        sh->do_missing_entry_self_heal = _gf_true;
-        sh->do_gfid_self_heal = _gf_true;
-        sh->do_data_self_heal = _gf_true;
-
-        reason = "subvolume came online";
-        afr_launch_self_heal (frame, this, inode, _gf_true, inode->ia_type,
-                              reason, NULL, NULL);
-}
-
-int
-afr_open_fd_fix (call_frame_t *frame, xlator_t *this, gf_boolean_t pause_fop)
-{
-        int                     ret = 0;
-        int                     i   = 0;
-        afr_fd_ctx_t            *fd_ctx = NULL;
-        inode_t                 *inode = NULL;
-        gf_boolean_t            need_self_heal = _gf_false;
-        int                     *need_open = NULL;
-        int                     need_open_count = 0;
-        afr_local_t             *local = NULL;
-        afr_private_t           *priv = NULL;
-        gf_boolean_t            fop_continue = _gf_true;
-
-        local = frame->local;
-        priv  = this->private;
-
-        GF_ASSERT (local->fd);
-
-        inode = local->fd->inode;
-        //gfid is not set in rebalance, that case needs to be handled.
-        if (fd_is_anonymous (local->fd) ||
-            !inode || uuid_is_null (inode->gfid)) {
-                fop_continue = _gf_true;
-                goto out;
-        }
-
-        if (pause_fop)
-                GF_ASSERT (local->fop_call_continue);
-
-        ret = afr_prepare_loc (frame, local->fd);
-        if (ret < 0) {
-                //File does not exist we cant open it.
-                ret = 0;
-                goto out;
-        }
-
-        fd_ctx = afr_fd_ctx_get (local->fd, this);
-        if (!fd_ctx) {
-                ret = -EINVAL;
-                goto out;
-        }
-
-        LOCK (&local->fd->lock);
-        {
-                if (fd_ctx->up_count < priv->up_count) {
-                        need_self_heal = _gf_true;
-                        fd_ctx->up_count   = priv->up_count;
-                        fd_ctx->down_count = priv->down_count;
-                }
-
-                for (i = 0; i < priv->child_count; i++) {
-                        if ((fd_ctx->opened_on[i] == AFR_FD_NOT_OPENED) &&
-                            local->child_up[i]) {
-                                fd_ctx->opened_on[i] = AFR_FD_OPENING;
-                                if (!need_open)
-                                        need_open = GF_CALLOC (priv->child_count,
-                                                               sizeof (*need_open),
-                                                               gf_afr_mt_int32_t);
-                                need_open[i] = 1;
-                                need_open_count++;
-                        } else if (pause_fop && local->child_up[i] &&
-                                   (fd_ctx->opened_on[i] == AFR_FD_OPENING)) {
-                                local->fop_paused = _gf_true;
-                        }
-                }
-
-                if (local->fop_paused) {
-                        GF_ASSERT (pause_fop);
-                        gf_log (this->name, GF_LOG_INFO, "Pause fd %p",
-                                local->fd);
-                        ret = afr_pause_fd_fop (frame, this, fd_ctx);
-                        if (ret)
-                                goto unlock;
-                        fop_continue = _gf_false;
-                }
-        }
-unlock:
-        UNLOCK (&local->fd->lock);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to fix fd for %s",
-                        local->loc.path);
-                fop_continue = _gf_false;
-                goto out;
-        }
-
-        if (need_self_heal)
-                afr_trigger_open_fd_self_heal (frame, this);
-
-        if (!need_open_count)
-                goto out;
-
-        gf_log (this->name, GF_LOG_INFO, "Opening fd %p", local->fd);
-        afr_fix_open (frame, this, fd_ctx, need_open_count, need_open);
-        fop_continue = _gf_false;
-out:
-        if (need_open)
-                GF_FREE (need_open);
-        if (fop_continue && local->fop_call_continue)
-                local->fop_call_continue (frame, this);
-        return ret;
-}
 
 int
 afr_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
             struct iovec *vector, int32_t count, off_t offset,
             uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        int ret = -1;
-        int op_errno = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+        afr_private_t *priv = NULL;
+        afr_local_t *local = NULL;
+        int op_errno = ENOMEM;
 
         priv = this->private;
 
         QUORUM_CHECK(writev,out);
 
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
 
-        local->cont.writev.vector     = iov_dup (vector, count);
+        local->cont.writev.vector = iov_dup (vector, count);
+	if (!local->cont.writev.vector)
+		goto out;
         local->cont.writev.count      = count;
         local->cont.writev.offset     = offset;
         local->cont.writev.flags      = flags;
         local->cont.writev.iobref     = iobref_ref (iobref);
 
-        local->fd                = fd_ref (fd);
-        local->fop_call_continue = afr_do_writev;
+	if (xdata)
+		local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
 
-        ret = afr_open_fd_fix (frame, this, _gf_true);
-        if (ret) {
-                op_errno = -ret;
-                goto out;
-        }
+	if (!local->xdata_req)
+		goto out;
+
+        local->fd = fd_ref (fd);
+	local->inode = inode_ref (fd->inode);
+
+	if (dict_set_uint32 (local->xdata_req, GLUSTERFS_OPEN_FD_COUNT, 4)) {
+		op_errno = ENOMEM;
+		goto out;
+	}
 
-        ret = 0;
+	if (dict_set_uint32 (local->xdata_req, GLUSTERFS_WRITE_IS_APPEND, 4)) {
+		op_errno = ENOMEM;
+		goto out;
+	}
+
+	/* Set append_write to be true speculatively. If on any
+	   server it turns not be true, we unset it in the
+	   callback.
+	*/
+	local->append_write = _gf_true;
+
+	/* detect here, but set it in writev_wind_cbk *after* the unstable
+	   write is performed
+	*/
+	local->stable_write = !!((fd->flags|flags)&(O_SYNC|O_DSYNC));
+
+        afr_fix_open (fd, this);
+
+        afr_do_writev (frame, this);
+
+	return 0;
 out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL);
+	AFR_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL);
 
         return 0;
 }
@@ -497,22 +479,13 @@ afr_truncate_unwind (call_frame_t *frame, xlator_t *this)
 
         local = frame->local;
 
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame)
-                        main_frame = local->transaction.main_frame;
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
-
-        if (main_frame) {
-                AFR_STACK_UNWIND (truncate, main_frame, local->op_ret,
-                                  local->op_errno,
-                                  &local->cont.truncate.prebuf,
-                                  &local->cont.truncate.postbuf,
-                                  NULL);
-        }
+	main_frame = afr_transaction_detach_fop_frame (frame);
+	if (!main_frame)
+		return 0;
 
+	AFR_STACK_UNWIND (truncate, main_frame, local->op_ret, local->op_errno,
+			  &local->cont.inode_wfop.prebuf,
+			  &local->cont.inode_wfop.postbuf, local->xdata_rsp);
         return 0;
 }
 
@@ -522,114 +495,32 @@ afr_truncate_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                        int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
                        struct iatt *postbuf, dict_t *xdata)
 {
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
-        int child_index = (long) cookie;
-        int read_child  = 0;
-        int call_count  = -1;
-        int need_unwind = 0;
-
-        local = frame->local;
-        priv  = this->private;
-
-        read_child = afr_inode_get_read_ctx (this, local->loc.inode, NULL);
+	afr_local_t *local = NULL;
 
-        LOCK (&frame->lock);
-        {
-                if (child_index == read_child) {
-                        local->read_child_returned = _gf_true;
-                }
+	local = frame->local;
 
-                if (afr_fop_failed (op_ret, op_errno) && op_errno != EFBIG)
-                        afr_transaction_fop_failed (frame, this, child_index);
+	if (op_ret == 0 && prebuf->ia_size != postbuf->ia_size)
+		local->stable_write = _gf_false;
 
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                                local->cont.truncate.prebuf  = *prebuf;
-                                local->cont.truncate.postbuf = *postbuf;
-                        }
-
-                        if (child_index == read_child) {
-                                local->cont.truncate.prebuf  = *prebuf;
-                                local->cont.truncate.postbuf = *postbuf;
-                        }
-
-                        local->success_count++;
-
-                        if ((local->success_count >= priv->wait_count)
-                            && local->read_child_returned) {
-                                need_unwind = 1;
-                        }
-                }
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
-
-        if (need_unwind)
-                local->transaction.unwind (frame, this);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-        }
-
-        return 0;
+	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
+				      prebuf, postbuf, xdata);
 }
 
 
-int32_t
-afr_truncate_wind (call_frame_t *frame, xlator_t *this)
+int
+afr_truncate_wind (call_frame_t *frame, xlator_t *this, int subvol)
 {
         afr_local_t *local = NULL;
         afr_private_t *priv = NULL;
-        int call_count = -1;
-        int i = 0;
 
         local = frame->local;
         priv = this->private;
 
-        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
-                                                     priv->child_count);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
-
-        local->call_count = call_count;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i]) {
-                        STACK_WIND_COOKIE (frame, afr_truncate_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->truncate,
-                                           &local->loc,
-                                           local->cont.truncate.offset,
-                                           NULL);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
-
-        return 0;
-}
-
-
-int
-afr_truncate_done (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local = NULL;
-
-        local = frame->local;
-
-        local->transaction.unwind (frame, this);
-
-        AFR_STACK_DESTROY (frame);
-
+	STACK_WIND_COOKIE (frame, afr_truncate_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->truncate,
+			   &local->loc, local->cont.truncate.offset,
+			   local->xdata_req);
         return 0;
 }
 
@@ -641,52 +532,60 @@ afr_truncate (call_frame_t *frame, xlator_t *this,
         afr_private_t * priv  = NULL;
         afr_local_t   * local = NULL;
         call_frame_t   *transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+	int ret = -1;
+        int op_errno = ENOMEM;
 
         priv = this->private;
 
         QUORUM_CHECK(truncate,out);
 
         transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                op_errno = ENOMEM;
+        if (!transaction_frame)
                 goto out;
-        }
-
-        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
-        local = transaction_frame->local;
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+	local = AFR_FRAME_INIT (transaction_frame, op_errno);
+	if (!local)
+		goto out;
 
         local->cont.truncate.offset  = offset;
+	if (xdata)
+		local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
+
+	if (!local->xdata_req)
+		goto out;
 
-        local->transaction.fop    = afr_truncate_wind;
-        local->transaction.done   = afr_truncate_done;
+	local->transaction.wind   = afr_truncate_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
         local->transaction.unwind = afr_truncate_unwind;
 
         loc_copy (&local->loc, loc);
+	local->inode = inode_ref (loc->inode);
+
+        local->op = GF_FOP_TRUNCATE;
 
         local->transaction.main_frame = frame;
         local->transaction.start   = offset;
         local->transaction.len     = 0;
 
-        afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
+	/* Set it true speculatively, will get reset in afr_truncate_wind_cbk
+	   if truncate was not a NOP */
+	local->stable_write = _gf_true;
 
-        ret = 0;
-out:
+        ret = afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
         if (ret < 0) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (truncate, frame, -1, op_errno, NULL, NULL, NULL);
+		op_errno = -ret;
+		goto out;
         }
 
+	return 0;
+out:
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
+
+	AFR_STACK_UNWIND (truncate, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
@@ -704,21 +603,13 @@ afr_ftruncate_unwind (call_frame_t *frame, xlator_t *this)
 
         local = frame->local;
 
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame)
-                        main_frame = local->transaction.main_frame;
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
+	main_frame = afr_transaction_detach_fop_frame (frame);
+	if (!main_frame)
+		return 0;
 
-        if (main_frame) {
-                AFR_STACK_UNWIND (ftruncate, main_frame, local->op_ret,
-                                  local->op_errno,
-                                  &local->cont.ftruncate.prebuf,
-                                  &local->cont.ftruncate.postbuf,
-                                  NULL);
-        }
+	AFR_STACK_UNWIND (ftruncate, main_frame, local->op_ret, local->op_errno,
+			  &local->cont.inode_wfop.prebuf,
+			  &local->cont.inode_wfop.postbuf, local->xdata_rsp);
         return 0;
 }
 
@@ -728,140 +619,75 @@ afr_ftruncate_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
                         struct iatt *postbuf, dict_t *xdata)
 {
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
-        int child_index = (long) cookie;
-        int call_count  = -1;
-        int need_unwind = 0;
-        int read_child  = 0;
-
-        local = frame->local;
-        priv  = this->private;
-
-        read_child = afr_inode_get_read_ctx (this, local->fd->inode, NULL);
-
-        LOCK (&frame->lock);
-        {
-                if (child_index == read_child) {
-                        local->read_child_returned = _gf_true;
-                }
-
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
-
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                                local->cont.ftruncate.prebuf  = *prebuf;
-                                local->cont.ftruncate.postbuf = *postbuf;
-                        }
-
-                        if (child_index == read_child) {
-                                local->cont.ftruncate.prebuf  = *prebuf;
-                                local->cont.ftruncate.postbuf = *postbuf;
-                        }
+	afr_local_t *local = NULL;
 
-                        local->success_count++;
+	local = frame->local;
 
-                        if ((local->success_count >= priv->wait_count)
-                            && local->read_child_returned) {
-                                need_unwind = 1;
-                        }
-                }
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
-
-        if (need_unwind)
-                local->transaction.unwind (frame, this);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-        }
+	if (op_ret == 0 && prebuf->ia_size != postbuf->ia_size)
+		local->stable_write = _gf_false;
 
-        return 0;
+	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
+				      prebuf, postbuf, xdata);
 }
 
 
 int
-afr_ftruncate_wind (call_frame_t *frame, xlator_t *this)
+afr_ftruncate_wind (call_frame_t *frame, xlator_t *this, int subvol)
 {
         afr_local_t *local = NULL;
         afr_private_t *priv = NULL;
-        int call_count = -1;
-        int i = 0;
-
-        local = frame->local;
-        priv = this->private;
-
-        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
-                                                     priv->child_count);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
-
-        local->call_count = call_count;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i]) {
-                        STACK_WIND_COOKIE (frame, afr_ftruncate_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->ftruncate,
-                                           local->fd,
-                                           local->cont.ftruncate.offset,
-                                           NULL);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
+	local = frame->local;
+	priv = this->private;
 
+	STACK_WIND_COOKIE (frame, afr_ftruncate_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->ftruncate,
+			   local->fd, local->cont.ftruncate.offset,
+			   local->xdata_req);
         return 0;
 }
 
 
 int
-afr_ftruncate_done (call_frame_t *frame, xlator_t *this)
+afr_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+	       dict_t *xdata)
 {
+        afr_private_t *priv = NULL;
         afr_local_t *local = NULL;
+	call_frame_t *transaction_frame = NULL;
+	int ret = -1;
+        int op_errno = ENOMEM;
 
-        local = frame->local;
-
-        local->transaction.unwind (frame, this);
-
-        AFR_STACK_DESTROY (frame);
+        priv = this->private;
 
-        return 0;
-}
+        QUORUM_CHECK(ftruncate,out);
 
+	transaction_frame = copy_frame (frame);
+	if (!frame)
+		goto out;
 
-int
-afr_do_ftruncate (call_frame_t *frame, xlator_t *this)
-{
-        call_frame_t * transaction_frame = NULL;
-        afr_local_t *  local             = NULL;
-        int op_ret   = -1;
-        int op_errno = 0;
+        local = AFR_FRAME_INIT (transaction_frame, op_errno);
+        if (!local)
+		goto out;
 
-        local = frame->local;
+        local->cont.ftruncate.offset  = offset;
+	if (xdata)
+		local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                goto out;
-        }
+	if (!local->xdata_req)
+		goto out;
 
-        transaction_frame->local = local;
-        frame->local = NULL;
+        local->fd = fd_ref (fd);
+	local->inode = inode_ref (fd->inode);
 
         local->op = GF_FOP_FTRUNCATE;
 
-        local->transaction.fop    = afr_ftruncate_wind;
-        local->transaction.done   = afr_ftruncate_done;
+	local->transaction.wind   = afr_ftruncate_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
         local->transaction.unwind = afr_ftruncate_unwind;
 
         local->transaction.main_frame = frame;
@@ -869,64 +695,21 @@ afr_do_ftruncate (call_frame_t *frame, xlator_t *this)
         local->transaction.start   = local->cont.ftruncate.offset;
         local->transaction.len     = 0;
 
-        afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
-
-        op_ret = 0;
-out:
-        if (op_ret == -1) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (ftruncate, frame, op_ret, op_errno, NULL,
-                                  NULL, NULL);
-        }
-
-        return 0;
-}
-
-
-int
-afr_ftruncate (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, off_t offset, dict_t *xdata)
-{
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t   *transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
-
-        priv = this->private;
-
-        QUORUM_CHECK(ftruncate,out);
-
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+        afr_fix_open (fd, this);
 
-        local->cont.ftruncate.offset  = offset;
+	/* Set it true speculatively, will get reset in afr_ftruncate_wind_cbk
+	   if truncate was not a NOP */
+	local->stable_write = _gf_true;
 
-        local->fd = fd_ref (fd);
-        local->fop_call_continue = afr_do_ftruncate;
-
-        ret = afr_open_fd_fix (frame, this, _gf_true);
-        if (ret) {
-                op_errno = -ret;
-                goto out;
+        ret = afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
+        if (ret < 0) {
+		op_errno = -ret;
+		goto out;
         }
 
-        ret = 0;
+	return 0;
 out:
-        if (ret < 0) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (ftruncate, frame, -1, op_errno, NULL, NULL, NULL);
-        }
+	AFR_STACK_UNWIND (ftruncate, frame, -1, op_errno, NULL, NULL, NULL);
 
         return 0;
 }
@@ -938,203 +721,109 @@ out:
 int
 afr_setattr_unwind (call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
+        afr_local_t *local = NULL;
+        call_frame_t *main_frame = NULL;
 
         local = frame->local;
 
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame)
-                        main_frame = local->transaction.main_frame;
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
-
-        if (main_frame) {
-                AFR_STACK_UNWIND (setattr, main_frame, local->op_ret,
-                                  local->op_errno,
-                                  &local->cont.setattr.preop_buf,
-                                  &local->cont.setattr.postop_buf,
-                                  NULL);
-        }
+	main_frame = afr_transaction_detach_fop_frame (frame);
+	if (!main_frame)
+		return 0;
 
+	AFR_STACK_UNWIND (setattr, main_frame, local->op_ret, local->op_errno,
+			  &local->cont.inode_wfop.prebuf,
+			  &local->cont.inode_wfop.postbuf,
+			  local->xdata_rsp);
         return 0;
 }
 
 
 int
 afr_setattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno,
+                      int op_ret, int op_errno,
                       struct iatt *preop, struct iatt *postop, dict_t *xdata)
 {
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
-        int child_index = (long) cookie;
-        int read_child  = 0;
-        int call_count  = -1;
-        int need_unwind = 0;
-
-        local = frame->local;
-        priv  = this->private;
-
-        read_child = afr_inode_get_read_ctx (this, local->loc.inode, NULL);
-
-        LOCK (&frame->lock);
-        {
-                if (child_index == read_child) {
-                        local->read_child_returned = _gf_true;
-                }
-
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
-
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                                local->cont.setattr.preop_buf  = *preop;
-                                local->cont.setattr.postop_buf = *postop;
-                        }
-
-                        if (child_index == read_child) {
-                                local->cont.setattr.preop_buf  = *preop;
-                                local->cont.setattr.postop_buf = *postop;
-                        }
-
-                        local->success_count++;
-
-                        if ((local->success_count >= priv->wait_count)
-                            && local->read_child_returned) {
-                                need_unwind = 1;
-                        }
-                }
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
-
-        if (need_unwind)
-                local->transaction.unwind (frame, this);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-        }
-
-        return 0;
+	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
+				      preop, postop, xdata);
 }
 
 
-int32_t
-afr_setattr_wind (call_frame_t *frame, xlator_t *this)
+int
+afr_setattr_wind (call_frame_t *frame, xlator_t *this, int subvol)
 {
         afr_local_t *local = NULL;
         afr_private_t *priv = NULL;
-        int call_count = -1;
-        int i = 0;
 
         local = frame->local;
         priv = this->private;
 
-        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
-                                                     priv->child_count);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
-
-        local->call_count = call_count;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i]) {
-                        STACK_WIND_COOKIE (frame, afr_setattr_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->setattr,
-                                           &local->loc,
-                                           &local->cont.setattr.in_buf,
-                                           local->cont.setattr.valid,
-                                           NULL);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
-
+	STACK_WIND_COOKIE (frame, afr_setattr_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->setattr,
+			   &local->loc, &local->cont.setattr.in_buf,
+			   local->cont.setattr.valid, local->xdata_req);
         return 0;
 }
 
 
 int
-afr_setattr_done (call_frame_t *frame, xlator_t *this)
+afr_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *buf,
+	     int32_t valid, dict_t *xdata)
 {
+        afr_private_t *priv = NULL;
         afr_local_t *local = NULL;
-
-        local = frame->local;
-
-        local->transaction.unwind (frame, this);
-
-        AFR_STACK_DESTROY (frame);
-
-        return 0;
-}
-
-
-int
-afr_setattr (call_frame_t *frame, xlator_t *this,
-             loc_t *loc, struct iatt *buf, int32_t valid, dict_t *xdata)
-{
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t   *transaction_frame = NULL;
+        call_frame_t *transaction_frame = NULL;
         int ret = -1;
-        int op_errno = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+        int op_errno = ENOMEM;
 
         priv = this->private;
 
         QUORUM_CHECK(setattr,out);
 
         transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                op_errno = ENOMEM;
+        if (!transaction_frame)
                 goto out;
-        }
 
-        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
-        local = transaction_frame->local;
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+	local = AFR_FRAME_INIT (transaction_frame, op_errno);
+	if (!local)
+		goto out;
 
         local->cont.setattr.in_buf = *buf;
         local->cont.setattr.valid  = valid;
+	if (xdata)
+		local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
+
+	if (!local->xdata_req)
+		goto out;
 
-        local->transaction.fop    = afr_setattr_wind;
-        local->transaction.done   = afr_setattr_done;
+        local->transaction.wind   = afr_setattr_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
         local->transaction.unwind = afr_setattr_unwind;
 
         loc_copy (&local->loc, loc);
+	local->inode = inode_ref (loc->inode);
+
+	local->op = GF_FOP_SETATTR;
 
         local->transaction.main_frame = frame;
         local->transaction.start   = LLONG_MAX - 1;
         local->transaction.len     = 0;
 
-        afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
-
-        ret = 0;
-out:
+        ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
         if (ret < 0) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (setattr, frame, -1, op_errno, NULL, NULL, NULL);
+		op_errno = -ret;
+		goto out;
         }
 
+	return 0;
+out:
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
+
+	AFR_STACK_UNWIND (setattr, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
@@ -1148,22 +837,13 @@ afr_fsetattr_unwind (call_frame_t *frame, xlator_t *this)
 
         local = frame->local;
 
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame)
-                        main_frame = local->transaction.main_frame;
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
-
-        if (main_frame) {
-                AFR_STACK_UNWIND (fsetattr, main_frame, local->op_ret,
-                                  local->op_errno,
-                                  &local->cont.fsetattr.preop_buf,
-                                  &local->cont.fsetattr.postop_buf,
-                                  NULL);
-        }
+	main_frame = afr_transaction_detach_fop_frame (frame);
+        if (!main_frame)
+		return 0;
 
+	AFR_STACK_UNWIND (fsetattr, main_frame, local->op_ret, local->op_errno,
+			  &local->cont.inode_wfop.prebuf,
+			  &local->cont.inode_wfop.postbuf, local->xdata_rsp);
         return 0;
 }
 
@@ -1173,178 +853,89 @@ afr_fsetattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                        int32_t op_ret, int32_t op_errno,
                        struct iatt *preop, struct iatt *postop, dict_t *xdata)
 {
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
-        int child_index = (long) cookie;
-        int read_child  = 0;
-        int call_count  = -1;
-        int need_unwind = 0;
-
-        local = frame->local;
-        priv  = this->private;
-
-        read_child = afr_inode_get_read_ctx (this, local->fd->inode, NULL);
-
-        LOCK (&frame->lock);
-        {
-                if (child_index == read_child) {
-                        local->read_child_returned = _gf_true;
-                }
-
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
-
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                                local->cont.fsetattr.preop_buf  = *preop;
-                                local->cont.fsetattr.postop_buf = *postop;
-                        }
-
-                        if (child_index == read_child) {
-                                local->cont.fsetattr.preop_buf  = *preop;
-                                local->cont.fsetattr.postop_buf = *postop;
-                        }
-
-                        local->success_count++;
-
-                        if ((local->success_count >= priv->wait_count)
-                            && local->read_child_returned) {
-                                need_unwind = 1;
-                        }
-                }
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
-
-        if (need_unwind)
-                local->transaction.unwind (frame, this);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-        }
-
-        return 0;
+	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
+				      preop, postop, xdata);
 }
 
 
-int32_t
-afr_fsetattr_wind (call_frame_t *frame, xlator_t *this)
+int
+afr_fsetattr_wind (call_frame_t *frame, xlator_t *this, int subvol)
 {
         afr_local_t *local = NULL;
         afr_private_t *priv = NULL;
-        int call_count = -1;
-        int i = 0;
 
         local = frame->local;
         priv = this->private;
 
-        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
-                                                     priv->child_count);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
-
-        local->call_count = call_count;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i]) {
-                        STACK_WIND_COOKIE (frame, afr_fsetattr_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->fsetattr,
-                                           local->fd,
-                                           &local->cont.fsetattr.in_buf,
-                                           local->cont.fsetattr.valid,
-                                           NULL);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
-
+	STACK_WIND_COOKIE (frame, afr_fsetattr_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->fsetattr,
+			   local->fd, &local->cont.fsetattr.in_buf,
+			   local->cont.fsetattr.valid, local->xdata_req);
         return 0;
 }
 
 
 int
-afr_fsetattr_done (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local = NULL;
-
-        local = frame->local;
-
-        local->transaction.unwind (frame, this);
-
-        AFR_STACK_DESTROY (frame);
-
-        return 0;
-}
-
-int
 afr_fsetattr (call_frame_t *frame, xlator_t *this,
               fd_t *fd, struct iatt *buf, int32_t valid, dict_t *xdata)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t   *transaction_frame = NULL;
+        afr_private_t *priv = NULL;
+        afr_local_t *local = NULL;
+        call_frame_t *transaction_frame = NULL;
         int ret = -1;
-        int op_errno = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+        int op_errno = ENOMEM;
 
         priv = this->private;
 
         QUORUM_CHECK(fsetattr,out);
 
         transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                op_errno = ENOMEM;
+        if (!transaction_frame)
                 goto out;
-        }
 
-        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
-        local = transaction_frame->local;
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+	local = AFR_FRAME_INIT (transaction_frame, op_errno);
+	if (!local)
+		goto out;
 
         local->cont.fsetattr.in_buf = *buf;
         local->cont.fsetattr.valid  = valid;
+	if (xdata)
+		local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
 
-        local->transaction.fop    = afr_fsetattr_wind;
-        local->transaction.done   = afr_fsetattr_done;
+	if (!local->xdata_req)
+		goto out;
+
+        local->transaction.wind   = afr_fsetattr_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
         local->transaction.unwind = afr_fsetattr_unwind;
 
         local->fd                 = fd_ref (fd);
+	local->inode = inode_ref (fd->inode);
 
-        ret = afr_open_fd_fix (transaction_frame, this, _gf_false);
-        if (ret) {
-                op_errno = -ret;
-                goto out;
-        }
+	local->op = GF_FOP_FSETATTR;
+
+        afr_fix_open (fd, this);
 
         local->transaction.main_frame = frame;
         local->transaction.start   = LLONG_MAX - 1;
         local->transaction.len     = 0;
 
-        afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
-
-        ret = 0;
-out:
+        ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
         if (ret < 0) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (fsetattr, frame, -1, op_errno, NULL, NULL, NULL);
+		op_errno = -ret;
+		goto out;
         }
 
+	return 0;
+out:
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
+
+	AFR_STACK_UNWIND (fsetattr, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
@@ -1360,19 +951,12 @@ afr_setxattr_unwind (call_frame_t *frame, xlator_t *this)
 
         local = frame->local;
 
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame)
-                        main_frame = local->transaction.main_frame;
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
+	main_frame = afr_transaction_detach_fop_frame (frame);
+	if (!main_frame)
+		return 0;
 
-        if (main_frame) {
-                AFR_STACK_UNWIND (setxattr, main_frame,
-                                  local->op_ret, local->op_errno,
-                                  NULL);
-        }
+	AFR_STACK_UNWIND (setxattr, main_frame, local->op_ret, local->op_errno,
+			  local->xdata_rsp);
         return 0;
 }
 
@@ -1381,158 +965,94 @@ int
 afr_setxattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        afr_local_t      *local         = NULL;
-        afr_private_t    *priv          = NULL;
-        int               call_count    = -1;
-        int               need_unwind   = 0;
-
-        local = frame->local;
-        priv = this->private;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                        }
-                        local->success_count++;
-
-                        if (local->success_count == priv->child_count) {
-                                need_unwind = 1;
-                        }
-                }
-
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
-
-        if (need_unwind)
-                local->transaction.unwind (frame, this);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-        }
-
-        return 0;
+        return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
+				      NULL, NULL, xdata);
 }
 
 
 int
-afr_setxattr_wind (call_frame_t *frame, xlator_t *this)
+afr_setxattr_wind (call_frame_t *frame, xlator_t *this, int subvol)
 {
         afr_local_t      *local         = NULL;
         afr_private_t    *priv          = NULL;
-        int               call_count    = -1;
-        int               i             = 0;
 
         local = frame->local;
         priv = this->private;
 
-        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
-                                                     priv->child_count);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
-
-        local->call_count = call_count;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i]) {
-                        STACK_WIND_COOKIE (frame, afr_setxattr_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->setxattr,
-                                           &local->loc,
-                                           local->cont.setxattr.dict,
-                                           local->cont.setxattr.flags,
-                                           NULL);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
-
+	STACK_WIND_COOKIE (frame, afr_setxattr_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->setxattr,
+			   &local->loc, local->cont.setxattr.dict,
+			   local->cont.setxattr.flags, local->xdata_req);
         return 0;
 }
 
 
 int
-afr_setxattr_done (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t   *local    = frame->local;
-
-        local->transaction.unwind (frame, this);
-
-        AFR_STACK_DESTROY (frame);
-
-        return 0;
-}
-
-int
-afr_setxattr (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, dict_t *dict, int32_t flags, dict_t *xdata)
+afr_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+	      int32_t flags, dict_t *xdata)
 {
         afr_private_t  *priv              = NULL;
         afr_local_t    *local             = NULL;
         call_frame_t   *transaction_frame = NULL;
-        data_pair_t    *trav              = NULL;
         int             ret               = -1;
         int             op_errno          = EINVAL;
 
-        VALIDATE_OR_GOTO (this, out);
-
         GF_IF_INTERNAL_XATTR_GOTO ("trusted.afr.*", dict,
-                                   trav, op_errno, out);
+                                   op_errno, out);
 
         GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.afr.*", dict,
-                                   trav, op_errno, out);
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this->private, out);
+                                   op_errno, out);
 
         priv = this->private;
 
         QUORUM_CHECK(setxattr,out);
+
         transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                op_errno = ENOMEM;
+        if (!transaction_frame)
                 goto out;
-        }
 
-        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
-        local = transaction_frame->local;
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
+	local = AFR_FRAME_INIT (transaction_frame, op_errno);
+	if (!local)
                 goto out;
 
         local->cont.setxattr.dict  = dict_ref (dict);
         local->cont.setxattr.flags = flags;
+	if (xdata)
+		local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
+
+	if (!local->xdata_req)
+		goto out;
 
-        local->transaction.fop    = afr_setxattr_wind;
-        local->transaction.done   = afr_setxattr_done;
+        local->transaction.wind   = afr_setxattr_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
         local->transaction.unwind = afr_setxattr_unwind;
 
         loc_copy (&local->loc, loc);
+	local->inode = inode_ref (loc->inode);
 
         local->transaction.main_frame = frame;
         local->transaction.start   = LLONG_MAX - 1;
         local->transaction.len     = 0;
 
-        afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+	local->op = GF_FOP_SETXATTR;
 
-        ret = 0;
-out:
+        ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
         if (ret < 0) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (setxattr, frame, -1, op_errno, NULL);
+		op_errno = -ret;
+		goto out;
         }
 
+	return 0;
+out:
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
+
+	AFR_STACK_UNWIND (setxattr, frame, -1, op_errno, NULL);
+
         return 0;
 }
 
@@ -1547,19 +1067,12 @@ afr_fsetxattr_unwind (call_frame_t *frame, xlator_t *this)
 
         local = frame->local;
 
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame)
-                        main_frame = local->transaction.main_frame;
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
+	main_frame = afr_transaction_detach_fop_frame (frame);
+        if (!main_frame)
+		return 0;
 
-        if (main_frame) {
-                AFR_STACK_UNWIND (fsetxattr, main_frame,
-                                  local->op_ret, local->op_errno,
-                                  NULL);
-        }
+	AFR_STACK_UNWIND (fsetxattr, main_frame, local->op_ret, local->op_errno,
+			  local->xdata_rsp);
         return 0;
 }
 
@@ -1568,98 +1081,30 @@ int
 afr_fsetxattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        afr_local_t     *local       = NULL;
-        afr_private_t   *priv        = NULL;
-        int              call_count  = -1;
-        int              need_unwind = 0;
-
-        local = frame->local;
-        priv = this->private;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                        }
-                        local->success_count++;
-
-                        if (local->success_count == priv->child_count) {
-                                need_unwind = 1;
-                        }
-                }
-
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
-
-        if (need_unwind)
-                local->transaction.unwind (frame, this);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-        }
-
-        return 0;
+        return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
+				      NULL, NULL, xdata);
 }
 
 
 int
-afr_fsetxattr_wind (call_frame_t *frame, xlator_t *this)
+afr_fsetxattr_wind (call_frame_t *frame, xlator_t *this, int subvol)
 {
         afr_local_t        *local       = NULL;
         afr_private_t      *priv        = NULL;
-        int                 call_count  = -1;
-        int                 i           = 0;
 
         local = frame->local;
         priv = this->private;
 
-        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
-                                                     priv->child_count);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
-
-        local->call_count = call_count;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i]) {
-                        STACK_WIND_COOKIE (frame, afr_fsetxattr_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->fsetxattr,
-                                           local->fd,
-                                           local->cont.fsetxattr.dict,
-                                           local->cont.fsetxattr.flags,
-                                           NULL);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
-
+	STACK_WIND_COOKIE (frame, afr_fsetxattr_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->fsetxattr,
+			   local->fd, local->cont.fsetxattr.dict,
+			   local->cont.fsetxattr.flags, local->xdata_req);
         return 0;
 }
 
 
 int
-afr_fsetxattr_done (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t   *local   = frame->local;
-
-        local->transaction.unwind (frame, this);
-
-        AFR_STACK_DESTROY (frame);
-
-        return 0;
-}
-
-int
 afr_fsetxattr (call_frame_t *frame, xlator_t *this,
                fd_t *fd, dict_t *dict, int32_t flags, dict_t *xdata)
 {
@@ -1667,64 +1112,63 @@ afr_fsetxattr (call_frame_t *frame, xlator_t *this,
         afr_local_t      *local             = NULL;
         call_frame_t     *transaction_frame = NULL;
         int               ret               = -1;
-        int               op_errno          = EINVAL;
-        data_pair_t      *trav              = NULL;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+        int               op_errno          = ENOMEM;
 
         GF_IF_INTERNAL_XATTR_GOTO ("trusted.afr.*", dict,
-                                   trav, op_errno, out);
+                                   op_errno, out);
 
         GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.afr.*", dict,
-                                   trav, op_errno, out);
-
-        if (ret)
-                goto out;
+                                   op_errno, out);
 
         priv = this->private;
 
         QUORUM_CHECK(fsetxattr,out);
 
-        AFR_LOCAL_ALLOC_OR_GOTO (local, out);
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
-
         transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
+        if (!transaction_frame)
                 goto out;
-        }
 
-        transaction_frame->local = local;
-
-        local->op_ret = -1;
+	local = AFR_FRAME_INIT (transaction_frame, op_errno);
+	if (!local)
+		goto out;
 
         local->cont.fsetxattr.dict  = dict_ref (dict);
         local->cont.fsetxattr.flags = flags;
 
-        local->transaction.fop    = afr_fsetxattr_wind;
-        local->transaction.done   = afr_fsetxattr_done;
+	if (xdata)
+		local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
+
+	if (!local->xdata_req)
+		goto out;
+
+        local->transaction.wind   = afr_fsetxattr_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
         local->transaction.unwind = afr_fsetxattr_unwind;
 
         local->fd                 = fd_ref (fd);
+	local->inode = inode_ref (fd->inode);
+
+	local->op = GF_FOP_FSETXATTR;
 
         local->transaction.main_frame = frame;
         local->transaction.start  = LLONG_MAX - 1;
         local->transaction.len    = 0;
 
-        afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
-
-        ret = 0;
-out:
+        ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
         if (ret < 0) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (fsetxattr, frame, -1, op_errno, NULL);
+		op_errno = -ret;
+		goto out;
         }
 
+	return 0;
+out:
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
+
+	AFR_STACK_UNWIND (fsetxattr, frame, -1, op_errno, NULL);
         return 0;
 }
 
@@ -1742,19 +1186,12 @@ afr_removexattr_unwind (call_frame_t *frame, xlator_t *this)
 
         local = frame->local;
 
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame)
-                        main_frame = local->transaction.main_frame;
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
+	main_frame = afr_transaction_detach_fop_frame (frame);
+	if (!main_frame)
+		return 0;
 
-        if (main_frame) {
-                AFR_STACK_UNWIND (removexattr, main_frame,
-                                  local->op_ret, local->op_errno,
-                                  NULL);
-        }
+	AFR_STACK_UNWIND (removexattr, main_frame, local->op_ret, local->op_errno,
+			  local->xdata_rsp);
         return 0;
 }
 
@@ -1763,108 +1200,151 @@ int
 afr_removexattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                           int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
-        int call_count  = -1;
-        int need_unwind = 0;
-
-        local = frame->local;
-        priv = this->private;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                        }
-                        local->success_count++;
-
-                        if (local->success_count == priv->wait_count) {
-                                need_unwind = 1;
-                        }
-                }
-
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
+        return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
+				      NULL, NULL, xdata);
+}
 
-        if (need_unwind)
-                local->transaction.unwind (frame, this);
 
-        call_count = afr_frame_return (frame);
+int
+afr_removexattr_wind (call_frame_t *frame, xlator_t *this, int subvol)
+{
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
 
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-        }
+	local = frame->local;
+	priv = this->private;
 
+	STACK_WIND_COOKIE (frame, afr_removexattr_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->removexattr,
+			   &local->loc, local->cont.removexattr.name,
+			   local->xdata_req);
         return 0;
 }
 
 
-int32_t
-afr_removexattr_wind (call_frame_t *frame, xlator_t *this)
+int
+afr_removexattr (call_frame_t *frame, xlator_t *this,
+                 loc_t *loc, const char *name, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        int call_count = -1;
-        int i = 0;
+        afr_private_t   *priv              = NULL;
+        afr_local_t     *local             = NULL;
+        call_frame_t    *transaction_frame = NULL;
+        int              ret               = -1;
+        int              op_errno          = ENOMEM;
+
+        GF_IF_NATIVE_XATTR_GOTO ("trusted.afr.*",
+                                 name, op_errno, out);
+
+        GF_IF_NATIVE_XATTR_GOTO ("trusted.glusterfs.afr.*",
+                                 name, op_errno, out);
 
-        local = frame->local;
         priv = this->private;
 
-        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
-                                                     priv->child_count);
+        QUORUM_CHECK(removexattr,out);
 
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
+        transaction_frame = copy_frame (frame);
+        if (!transaction_frame)
+                goto out;
 
-        local->call_count = call_count;
+	local = AFR_FRAME_INIT (transaction_frame, op_errno);
+	if (!local)
+		goto out;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i]) {
-                        STACK_WIND_COOKIE (frame, afr_removexattr_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->removexattr,
-                                           &local->loc,
-                                           local->cont.removexattr.name,
-                                           NULL);
-
-                        if (!--call_count)
-                                break;
-                }
+        local->cont.removexattr.name = gf_strdup (name);
+
+	if (xdata)
+		local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
+
+	if (!local->xdata_req)
+		goto out;
+
+        local->transaction.wind   = afr_removexattr_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
+        local->transaction.unwind = afr_removexattr_unwind;
+
+        loc_copy (&local->loc, loc);
+	local->inode = inode_ref (loc->inode);
+
+	local->op = GF_FOP_REMOVEXATTR;
+
+        local->transaction.main_frame = frame;
+        local->transaction.start   = LLONG_MAX - 1;
+        local->transaction.len     = 0;
+
+        ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+        if (ret < 0) {
+		op_errno = -ret;
+		goto out;
         }
 
+	return 0;
+out:
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
+
+	AFR_STACK_UNWIND (removexattr, frame, -1, op_errno, NULL);
         return 0;
 }
 
-
+/* ffremovexattr */
 int
-afr_removexattr_done (call_frame_t *frame, xlator_t *this)
+afr_fremovexattr_unwind (call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t * local = frame->local;
+        afr_local_t *   local = NULL;
+        call_frame_t   *main_frame = NULL;
 
-        local->transaction.unwind (frame, this);
+        local = frame->local;
 
-        AFR_STACK_DESTROY (frame);
+	main_frame = afr_transaction_detach_fop_frame (frame);
+        if (!main_frame)
+		return 0;
 
+	AFR_STACK_UNWIND (fremovexattr, main_frame, local->op_ret, local->op_errno,
+			  local->xdata_rsp);
         return 0;
 }
 
 
 int
-afr_removexattr (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc, const char *name, dict_t *xdata)
+afr_fremovexattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        afr_private_t   *priv              = NULL;
-        afr_local_t     *local             = NULL;
-        call_frame_t    *transaction_frame = NULL;
-        int              ret               = -1;
-        int              op_errno          = 0;
+	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
+				      NULL, NULL, xdata);
+}
 
-        VALIDATE_OR_GOTO (this, out);
+
+int
+afr_fremovexattr_wind (call_frame_t *frame, xlator_t *this, int subvol)
+{
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
+
+        local = frame->local;
+        priv = this->private;
+
+	STACK_WIND_COOKIE (frame, afr_fremovexattr_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->fremovexattr,
+			   local->fd, local->cont.removexattr.name,
+			   local->xdata_req);
+        return 0;
+}
+
+
+int
+afr_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+		  const char *name, dict_t *xdata)
+{
+	afr_private_t *priv = NULL;
+        afr_local_t *local = NULL;
+        call_frame_t *transaction_frame = NULL;
+        int ret = -1;
+        int op_errno = ENOMEM;
 
         GF_IF_NATIVE_XATTR_GOTO ("trusted.afr.*",
                                  name, op_errno, out);
@@ -1872,237 +1352,403 @@ afr_removexattr (call_frame_t *frame, xlator_t *this,
         GF_IF_NATIVE_XATTR_GOTO ("trusted.glusterfs.afr.*",
                                  name, op_errno, out);
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this->private, out);
-        VALIDATE_OR_GOTO (loc, out);
+	priv = this->private;
 
-        priv = this->private;
-
-        QUORUM_CHECK(removexattr,out);
+        QUORUM_CHECK(fremovexattr, out);
 
         transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                op_errno = ENOMEM;
+        if (!transaction_frame)
                 goto out;
-        }
-
-        AFR_LOCAL_ALLOC_OR_GOTO (transaction_frame->local, out);
-        local = transaction_frame->local;
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
+	local = AFR_FRAME_INIT (transaction_frame, op_errno);
+	if (!local)
                 goto out;
 
         local->cont.removexattr.name = gf_strdup (name);
+	if (xdata)
+		local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
 
-        local->transaction.fop    = afr_removexattr_wind;
-        local->transaction.done   = afr_removexattr_done;
-        local->transaction.unwind = afr_removexattr_unwind;
+	if (!local->xdata_req)
+		goto out;
 
-        loc_copy (&local->loc, loc);
+        local->transaction.wind   = afr_fremovexattr_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
+        local->transaction.unwind = afr_fremovexattr_unwind;
+
+        local->fd = fd_ref (fd);
+	local->inode = inode_ref (fd->inode);
+
+	local->op = GF_FOP_FREMOVEXATTR;
 
         local->transaction.main_frame = frame;
         local->transaction.start   = LLONG_MAX - 1;
         local->transaction.len     = 0;
 
-        afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
-
-        ret = 0;
-out:
+        ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
         if (ret < 0) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (removexattr, frame, -1, op_errno, NULL);
+		op_errno = -ret;
+		goto out;
         }
 
+	return 0;
+out:
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
+
+	AFR_STACK_UNWIND (fremovexattr, frame, -1, op_errno, NULL);
+
         return 0;
 }
 
-/* ffremovexattr */
+
 int
-afr_fremovexattr_unwind (call_frame_t *frame, xlator_t *this)
+afr_fallocate_unwind (call_frame_t *frame, xlator_t *this)
 {
         afr_local_t *   local = NULL;
         call_frame_t   *main_frame = NULL;
 
         local = frame->local;
 
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame)
-                        main_frame = local->transaction.main_frame;
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
+	main_frame = afr_transaction_detach_fop_frame (frame);
+	if (!main_frame)
+		return 0;
 
-        if (main_frame) {
-                AFR_STACK_UNWIND (fremovexattr, main_frame,
-                                  local->op_ret, local->op_errno,
-                                  NULL);
-        }
+	AFR_STACK_UNWIND (fallocate, main_frame, local->op_ret, local->op_errno,
+			  &local->cont.inode_wfop.prebuf,
+			  &local->cont.inode_wfop.postbuf, local->xdata_rsp);
         return 0;
 }
 
 
 int
-afr_fremovexattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
+afr_fallocate_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                        struct iatt *postbuf, dict_t *xdata)
 {
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
-        int call_count  = -1;
-        int need_unwind = 0;
+        return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
+				      prebuf, postbuf, xdata);
+}
+
+
+int
+afr_fallocate_wind (call_frame_t *frame, xlator_t *this, int subvol)
+{
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
 
         local = frame->local;
         priv = this->private;
 
-        LOCK (&frame->lock);
-        {
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                        }
-                        local->success_count++;
-
-                        if (local->success_count == priv->wait_count) {
-                                need_unwind = 1;
-                        }
-                }
+	STACK_WIND_COOKIE (frame, afr_fallocate_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->fallocate,
+			   local->fd, local->cont.fallocate.mode,
+			   local->cont.fallocate.offset,
+			   local->cont.fallocate.len, local->xdata_req);
+        return 0;
+}
 
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
 
-        if (need_unwind)
-                local->transaction.unwind (frame, this);
+int
+afr_fallocate (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+               off_t offset, size_t len, dict_t *xdata)
+{
+	afr_private_t *priv = NULL;
+        call_frame_t *transaction_frame = NULL;
+        afr_local_t *local = NULL;
+        int ret = -1;
+        int op_errno = ENOMEM;
 
-        call_count = afr_frame_return (frame);
+	priv = this->private;
 
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
+        QUORUM_CHECK(fallocate,out);
+
+        transaction_frame = copy_frame (frame);
+        if (!transaction_frame)
+                goto out;
+
+	local = AFR_FRAME_INIT (transaction_frame, op_errno);
+	if (!local)
+		goto out;
+
+        local->cont.fallocate.mode = mode;
+        local->cont.fallocate.offset  = offset;
+        local->cont.fallocate.len = len;
+
+        local->fd = fd_ref (fd);
+	local->inode = inode_ref (fd->inode);
+
+	if (xdata)
+		local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
+
+	if (!local->xdata_req)
+		goto out;
+
+        local->op = GF_FOP_FALLOCATE;
+
+        local->transaction.wind   = afr_fallocate_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
+        local->transaction.unwind = afr_fallocate_unwind;
+
+        local->transaction.main_frame = frame;
+
+        local->transaction.start   = local->cont.fallocate.offset;
+        local->transaction.len     = 0;
+
+        afr_fix_open (fd, this);
+
+        ret = afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
+        if (ret < 0) {
+		op_errno = -ret;
+		goto out;
         }
 
+	return 0;
+out:
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
+
+	AFR_STACK_UNWIND (fallocate, frame, -1, op_errno, NULL, NULL, NULL);
+        return 0;
+}
+
+
+/* }}} */
+
+/* {{{ discard */
+
+int
+afr_discard_unwind (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t *   local = NULL;
+        call_frame_t   *main_frame = NULL;
+
+        local = frame->local;
+
+	main_frame = afr_transaction_detach_fop_frame (frame);
+        if (!main_frame)
+		return 0;
+
+	AFR_STACK_UNWIND (discard, main_frame, local->op_ret, local->op_errno,
+			  &local->cont.inode_wfop.prebuf,
+			  &local->cont.inode_wfop.postbuf, local->xdata_rsp);
         return 0;
 }
 
 
-int32_t
-afr_fremovexattr_wind (call_frame_t *frame, xlator_t *this)
+int
+afr_discard_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xdata)
+{
+	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
+				      prebuf, postbuf, xdata);
+}
+
+
+int
+afr_discard_wind (call_frame_t *frame, xlator_t *this, int subvol)
 {
         afr_local_t *local = NULL;
         afr_private_t *priv = NULL;
-        int call_count = -1;
-        int i = 0;
 
         local = frame->local;
         priv = this->private;
 
-        call_count = afr_pre_op_done_children_count (local->transaction.pre_op,
-                                                     priv->child_count);
+	STACK_WIND_COOKIE (frame, afr_discard_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->discard,
+			   local->fd, local->cont.discard.offset,
+			   local->cont.discard.len, local->xdata_req);
+        return 0;
+}
 
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
 
-        local->call_count = call_count;
+int
+afr_discard (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             size_t len, dict_t *xdata)
+{
+        afr_private_t *priv  = NULL;
+        afr_local_t *local = NULL;
+        call_frame_t *transaction_frame = NULL;
+        int ret = -1;
+        int op_errno = ENOMEM;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i]) {
-                        STACK_WIND_COOKIE (frame, afr_fremovexattr_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->fremovexattr,
-                                           local->fd,
-                                           local->cont.removexattr.name,
-                                           NULL);
-
-                        if (!--call_count)
-                                break;
-                }
+        priv = this->private;
+
+        QUORUM_CHECK(discard, out);
+
+	transaction_frame = copy_frame (frame);
+	if (!transaction_frame)
+		goto out;
+
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
+
+        local->cont.discard.offset  = offset;
+        local->cont.discard.len = len;
+
+        local->fd = fd_ref (fd);
+	local->inode = inode_ref (fd->inode);
+
+	if (xdata)
+		local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
+
+	if (!local->xdata_req)
+		goto out;
+
+        local->op = GF_FOP_DISCARD;
+
+        local->transaction.wind   = afr_discard_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
+        local->transaction.unwind = afr_discard_unwind;
+
+        local->transaction.main_frame = frame;
+
+        local->transaction.start   = local->cont.discard.offset;
+        local->transaction.len     = 0;
+
+        afr_fix_open (fd, this);
+
+        ret = afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
+        if (ret < 0) {
+		op_errno = -ret;
+		goto out;
         }
 
+	return 0;
+out:
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
+
+	AFR_STACK_UNWIND (discard, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
 
+/* {{{ zerofill */
+
 int
-afr_fremovexattr_done (call_frame_t *frame, xlator_t *this)
+afr_zerofill_unwind (call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t * local = frame->local;
+        afr_local_t *   local = NULL;
+        call_frame_t   *main_frame = NULL;
 
-        local->transaction.unwind (frame, this);
+        local = frame->local;
 
-        AFR_STACK_DESTROY (frame);
+	main_frame = afr_transaction_detach_fop_frame (frame);
+        if (!main_frame)
+		return 0;
 
+	AFR_STACK_UNWIND (discard, main_frame, local->op_ret, local->op_errno,
+			  &local->cont.inode_wfop.prebuf,
+			  &local->cont.inode_wfop.postbuf, local->xdata_rsp);
         return 0;
 }
 
 
 int
-afr_fremovexattr (call_frame_t *frame, xlator_t *this,
-                  fd_t *fd, const char *name, dict_t *xdata)
+afr_zerofill_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xdata)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t   *transaction_frame = NULL;
-        int ret = -1;
-        int op_ret   = -1;
-        int op_errno = 0;
+	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
+				      prebuf, postbuf, xdata);
+}
 
-        VALIDATE_OR_GOTO (this, out);
 
-        GF_IF_NATIVE_XATTR_GOTO ("trusted.afr.*",
-                                 name, op_errno, out);
+int
+afr_zerofill_wind (call_frame_t *frame, xlator_t *this, int subvol)
+{
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
 
-        GF_IF_NATIVE_XATTR_GOTO ("trusted.glusterfs.afr.*",
-                                 name, op_errno, out);
+        local = frame->local;
+        priv = this->private;
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this->private, out);
+	STACK_WIND_COOKIE (frame, afr_zerofill_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->zerofill,
+			   local->fd, local->cont.zerofill.offset,
+			   local->cont.zerofill.len, local->xdata_req);
+        return 0;
+}
+
+int
+afr_zerofill (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             size_t len, dict_t *xdata)
+{
+        afr_private_t *priv  = NULL;
+        afr_local_t *local = NULL;
+        call_frame_t *transaction_frame = NULL;
+        int ret = -1;
+        int op_errno = ENOMEM;
 
         priv = this->private;
 
-        QUORUM_CHECK(fremovexattr, out);
+        QUORUM_CHECK(discard, out);
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                goto out;
-        }
+	transaction_frame = copy_frame (frame);
+	if (!transaction_frame)
+		goto out;
 
-        AFR_LOCAL_ALLOC_OR_GOTO (local, out);
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
-        }
+        local->cont.zerofill.offset  = offset;
+        local->cont.zerofill.len = len;
 
-        transaction_frame->local = local;
+        local->fd = fd_ref (fd);
+	local->inode = inode_ref (fd->inode);
 
-        local->op_ret = -1;
+	if (xdata)
+		local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
 
-        local->cont.removexattr.name = gf_strdup (name);
+	if (!local->xdata_req)
+		goto out;
 
-        local->transaction.fop    = afr_fremovexattr_wind;
-        local->transaction.done   = afr_fremovexattr_done;
-        local->transaction.unwind = afr_fremovexattr_unwind;
+        local->op = GF_FOP_ZEROFILL;
 
-        local->fd = fd_ref (fd);
+        local->transaction.wind   = afr_zerofill_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
+        local->transaction.unwind = afr_zerofill_unwind;
 
         local->transaction.main_frame = frame;
-        local->transaction.start   = LLONG_MAX - 1;
-        local->transaction.len     = 0;
 
-        afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+        local->transaction.start   = local->cont.discard.offset;
+        local->transaction.len     = len;
 
-        op_ret = 0;
-out:
-        if (op_ret == -1) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (fremovexattr, frame, op_ret, op_errno, NULL);
+        afr_fix_open (fd, this);
+
+        ret = afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
+        if (ret < 0) {
+		op_errno = -ret;
+		goto out;
         }
 
+	return 0;
+out:
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
+
+	AFR_STACK_UNWIND (zerofill, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
+
+/* }}} */
+
+
diff --git a/xlators/cluster/afr/src/afr-inode-write.h b/xlators/cluster/afr/src/afr-inode-write.h
index ed11079fd..7b1fc5528 100644
--- a/xlators/cluster/afr/src/afr-inode-write.h
+++ b/xlators/cluster/afr/src/afr-inode-write.h
@@ -68,4 +68,15 @@ int32_t
 afr_fremovexattr (call_frame_t *frame, xlator_t *this,
                   fd_t *fd, const char *name, dict_t *xdata);
 
+int
+afr_discard (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             size_t len, dict_t *xdata);
+
+int
+afr_fallocate (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+               off_t offset, size_t len, dict_t *xdata);
+
+int
+afr_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             off_t len, dict_t *xdata);
 #endif /* __INODE_WRITE_H__ */
diff --git a/xlators/cluster/afr/src/afr-lk-common.c b/xlators/cluster/afr/src/afr-lk-common.c
index 6fb8a2bc9..a2a758f35 100644
--- a/xlators/cluster/afr/src/afr-lk-common.c
+++ b/xlators/cluster/afr/src/afr-lk-common.c
@@ -55,7 +55,33 @@
         } while (0);
 
 int
-afr_lock_blocking (call_frame_t *frame, xlator_t *this, int child_index);
+afr_entry_lockee_cmp (const void *l1, const void *l2)
+{
+        const afr_entry_lockee_t       *r1 = l1;
+        const afr_entry_lockee_t       *r2 = l2;
+        int                            ret = 0;
+        uuid_t                         gfid1 = {0};
+        uuid_t                         gfid2 = {0};
+
+        loc_gfid ((loc_t*)&r1->loc, gfid1);
+        loc_gfid ((loc_t*)&r2->loc, gfid2);
+        ret = uuid_compare (gfid1, gfid2);
+        /*Entrylks with NULL basename are the 'smallest'*/
+        if (ret == 0) {
+                if (!r1->basename)
+                        return -1;
+                if (!r2->basename)
+                        return 1;
+                ret = strcmp (r1->basename, r2->basename);
+        }
+
+        if (ret <= 0)
+                return -1;
+        else
+                return 1;
+}
+
+int afr_lock_blocking (call_frame_t *frame, xlator_t *this, int child_index);
 
 static int
 afr_copy_locked_nodes (call_frame_t *frame, xlator_t *this);
@@ -125,16 +151,9 @@ internal_lock_count (call_frame_t *frame, xlator_t *this)
         local = frame->local;
         priv  = this->private;
 
-        if (local->fd) {
-                for (i = 0; i < priv->child_count; i++) {
-                        if (local->child_up[i] && local->fd_open_on[i])
-                                ++call_count;
-                }
-        } else {
-                for (i = 0; i < priv->child_count; i++) {
-                        if (local->child_up[i])
-                                ++call_count;
-                }
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->child_up[i])
+                        ++call_count;
         }
 
         return call_count;
@@ -332,10 +351,13 @@ static void
 afr_trace_entrylk_in (call_frame_t *frame, xlator_t *this,
                       afr_lock_call_type_t lock_call_type,
                       afr_lock_op_type_t lk_op_type, const char *basename,
-                      int32_t child_index)
+                      int32_t cookie)
 {
         afr_local_t         *local    = NULL;
         afr_internal_lock_t *int_lock = NULL;
+        afr_private_t       *priv     = NULL;
+        int                 child_index = 0;
+        int                 lockee_no = 0;
 
         char lock[256];
         char lockee[256];
@@ -343,28 +365,40 @@ afr_trace_entrylk_in (call_frame_t *frame, xlator_t *this,
 
         local    = frame->local;
         int_lock = &local->internal_lock;
+        priv     = this->private;
+
+        if (!priv->entrylk_trace) {
+                return;
+        }
+        lockee_no = cookie / priv->child_count;
+        child_index = cookie % priv->child_count;
 
         afr_print_entrylk (lock, 256, basename, &frame->root->lk_owner);
-        afr_print_lockee (lockee, 256, &local->loc, local->fd, child_index);
+        afr_print_lockee (lockee, 256, &int_lock->lockee[lockee_no].loc, local->fd,
+                          child_index);
 
         afr_set_lock_call_type (lock_call_type, lock_call_type_str, int_lock);
 
         gf_log (this->name, GF_LOG_INFO,
-                "[%s %s] Lock={%s} Lockee={%s} Number={%llu}",
+                "[%s %s] Lock={%s} Lockee={%s} Number={%llu}, Cookie={%d}",
                 lock_call_type_str,
                 lk_op_type == AFR_LOCK_OP ? "LOCK REQUEST" : "UNLOCK REQUEST",
                 lock, lockee,
-                (unsigned long long) int_lock->lock_number);
+                (unsigned long long) int_lock->lock_number,
+                cookie);
 }
 
 static void
 afr_trace_entrylk_out (call_frame_t *frame, xlator_t *this,
                        afr_lock_call_type_t lock_call_type,
                        afr_lock_op_type_t lk_op_type, const char *basename,
-                       int op_ret, int op_errno, int32_t child_index)
+                       int op_ret, int op_errno, int32_t cookie)
 {
         afr_internal_lock_t *int_lock = NULL;
         afr_local_t         *local    = NULL;
+        afr_private_t       *priv     = NULL;
+        int                 lockee_no = 0;
+        int                 child_index = 0;
 
         char lock[256];
         char lockee[256];
@@ -373,20 +407,30 @@ afr_trace_entrylk_out (call_frame_t *frame, xlator_t *this,
 
         local    = frame->local;
         int_lock = &local->internal_lock;
+        priv     = this->private;
 
-        afr_print_lockee (lockee, 256, &local->loc, local->fd, child_index);
+        if (!priv->entrylk_trace) {
+                return;
+        }
+        lockee_no = cookie / priv->child_count;
+        child_index = cookie % priv->child_count;
+
+        afr_print_entrylk (lock, 256, basename, &frame->root->lk_owner);
+        afr_print_lockee (lockee, 256, &int_lock->lockee[lockee_no].loc, local->fd,
+                          child_index);
 
         afr_set_lock_call_type (lock_call_type, lock_call_type_str, int_lock);
 
         afr_print_verdict (op_ret, op_errno, verdict);
 
         gf_log (this->name, GF_LOG_INFO,
-                "[%s %s] [%s] Lock={%s} Lockee={%s} Number={%llu}",
+                "[%s %s] [%s] Lock={%s} Lockee={%s} Number={%llu} Cookie={%d}",
                 lock_call_type_str,
                 lk_op_type == AFR_LOCK_OP ? "LOCK REPLY" : "UNLOCK REPLY",
                 verdict,
                 lock, lockee,
-                (unsigned long long) int_lock->lock_number);
+                (unsigned long long) int_lock->lock_number,
+                cookie);
 
 }
 
@@ -439,6 +483,47 @@ is_afr_lock_transaction (afr_local_t *local)
         return ret;
 }
 
+int
+afr_init_entry_lockee (afr_entry_lockee_t *lockee, afr_local_t *local,
+                       loc_t *loc, char *basename, int child_count)
+{
+        int     ret     = -1;
+
+        loc_copy (&lockee->loc, loc);
+        lockee->basename        = (basename)? gf_strdup (basename): NULL;
+        if (basename && !lockee->basename)
+                goto out;
+
+        lockee->locked_count    = 0;
+        lockee->locked_nodes    = GF_CALLOC (child_count,
+                                             sizeof (*lockee->locked_nodes),
+                                             gf_afr_mt_afr_node_character);
+
+        if (!lockee->locked_nodes)
+                goto out;
+
+        ret = 0;
+out:
+        return ret;
+
+}
+
+void
+afr_entry_lockee_cleanup (afr_internal_lock_t *int_lock)
+{
+        int     i   = 0;
+
+        for (i = 0; i < int_lock->lockee_count; i++) {
+                loc_wipe (&int_lock->lockee[i].loc);
+                if (int_lock->lockee[i].basename)
+                        GF_FREE (int_lock->lockee[i].basename);
+                if (int_lock->lockee[i].locked_nodes)
+                       GF_FREE (int_lock->lockee[i].locked_nodes);
+        }
+
+        return;
+}
+
 static int
 initialize_entrylk_variables (call_frame_t *frame, xlator_t *this)
 {
@@ -456,8 +541,13 @@ initialize_entrylk_variables (call_frame_t *frame, xlator_t *this)
         int_lock->lock_op_ret        = -1;
         int_lock->lock_op_errno      = 0;
 
-        for (i = 0; i < priv->child_count; i++) {
-                int_lock->entry_locked_nodes[i] = 0;
+        for (i = 0; i < AFR_LOCKEE_COUNT_MAX; i++) {
+                if (!int_lock->lockee[i].locked_nodes)
+                        break;
+                int_lock->lockee[i].locked_count = 0;
+                memset (int_lock->lockee[i].locked_nodes, 0,
+                        sizeof (*int_lock->lockee[i].locked_nodes) *
+                        priv->child_count);
         }
 
         return 0;
@@ -469,37 +559,37 @@ initialize_inodelk_variables (call_frame_t *frame, xlator_t *this)
         afr_local_t         *local    = NULL;
         afr_internal_lock_t *int_lock = NULL;
         afr_private_t       *priv     = NULL;
-        int i = 0;
+        afr_inodelk_t       *inodelk  = NULL;
 
         priv     = this->private;
         local    = frame->local;
         int_lock = &local->internal_lock;
 
-        int_lock->inodelk_lock_count = 0;
-        int_lock->lock_op_ret        = -1;
-        int_lock->lock_op_errno      = 0;
+        inodelk = afr_get_inodelk (int_lock, int_lock->domain);
 
-        for (i = 0; i < priv->child_count; i++) {
-                int_lock->inode_locked_nodes[i] = 0;
-        }
+        inodelk->lock_count    = 0;
+        int_lock->lk_attempted_count = 0;
+        int_lock->lock_op_ret   = -1;
+        int_lock->lock_op_errno = 0;
+
+        memset (inodelk->locked_nodes, 0,
+                sizeof (*inodelk->locked_nodes) * priv->child_count);
+        memset (int_lock->locked_nodes, 0,
+                sizeof (*int_lock->locked_nodes) * priv->child_count);
 
         return 0;
 }
 
-loc_t *
-lower_path (loc_t *l1, const char *b1, loc_t *l2, const char *b2)
+int
+afr_lockee_locked_nodes_count (afr_internal_lock_t *int_lock)
 {
-        int ret = 0;
-
-        ret = uuid_compare (l1->inode->gfid, l2->inode->gfid);
+        int call_count  = 0;
+        int i           = 0;
 
-        if (ret == 0)
-                ret = strcmp (b1, b2);
+        for (i = 0; i < int_lock->lockee_count; i++)
+                call_count += int_lock->lockee[i].locked_count;
 
-        if (ret <= 0)
-                return l1;
-        else
-                return l2;
+        return call_count;
 }
 
 int
@@ -550,7 +640,9 @@ afr_unlock_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 {
         afr_local_t         *local = NULL;
         afr_internal_lock_t *int_lock = NULL;
+        afr_inodelk_t       *inodelk = NULL;
         int32_t             child_index = (long)cookie;
+        afr_private_t       *priv = NULL;
 
         local = frame->local;
         int_lock = &local->internal_lock;
@@ -559,14 +651,18 @@ afr_unlock_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                AFR_UNLOCK_OP, NULL, op_ret,
                                op_errno, child_index);
 
+        priv = this->private;
+
         if (op_ret < 0 && op_errno != ENOTCONN && op_errno != EBADFD) {
-                gf_log (this->name, GF_LOG_INFO, "%s: unlock failed on %d "
-                        "unlock by %s", local->loc.path, child_index,
+                gf_log (this->name, GF_LOG_INFO, "%s: unlock failed on subvolume %s "
+                        "with lock owner %s", local->loc.path,
+                        priv->children[child_index]->name,
                         lkowner_utoa (&frame->root->lk_owner));
         }
 
 
-        int_lock->inode_locked_nodes[child_index] &= LOCKED_NO;
+        inodelk = afr_get_inodelk (int_lock, int_lock->domain);
+        inodelk->locked_nodes[child_index] &= LOCKED_NO;
         if (local->transaction.eager_lock)
                 local->transaction.eager_lock[child_index] = 0;
 
@@ -580,6 +676,7 @@ static int
 afr_unlock_inodelk (call_frame_t *frame, xlator_t *this)
 {
         afr_internal_lock_t *int_lock = NULL;
+        afr_inodelk_t       *inodelk  = NULL;
         afr_local_t         *local    = NULL;
         afr_private_t       *priv     = NULL;
         struct gf_flock flock = {0,};
@@ -589,19 +686,20 @@ afr_unlock_inodelk (call_frame_t *frame, xlator_t *this)
         int i = 0;
         int piggyback = 0;
         afr_fd_ctx_t        *fd_ctx      = NULL;
-        gf_boolean_t    fd_lock_owner = _gf_false;
 
 
         local    = frame->local;
         int_lock = &local->internal_lock;
         priv     = this->private;
 
-        flock.l_start = int_lock->lk_flock.l_start;
-        flock.l_len   = int_lock->lk_flock.l_len;
+        inodelk = afr_get_inodelk (int_lock, int_lock->domain);
+
+        flock.l_start = inodelk->flock.l_start;
+        flock.l_len   = inodelk->flock.l_len;
         flock.l_type  = F_UNLCK;
 
         full_flock.l_type = F_UNLCK;
-        call_count = afr_locked_nodes_count (int_lock->inode_locked_nodes,
+        call_count = afr_locked_nodes_count (inodelk->locked_nodes,
                                              priv->child_count);
 
         int_lock->lk_call_count = call_count;
@@ -617,18 +715,12 @@ afr_unlock_inodelk (call_frame_t *frame, xlator_t *this)
                 fd_ctx = afr_fd_ctx_get (local->fd, this);
 
         for (i = 0; i < priv->child_count; i++) {
-                if ((int_lock->inode_locked_nodes[i] & LOCKED_YES)
-                    != LOCKED_YES)
+                if ((inodelk->locked_nodes[i] & LOCKED_YES) != LOCKED_YES)
                         continue;
 
                 if (local->fd) {
                         flock_use = &flock;
                         if (!local->transaction.eager_lock[i]) {
-                                if (fd_lock_owner) {
-                                        afr_set_lk_owner (frame, this,
-                                                          frame->root);
-                                        fd_lock_owner = _gf_false;
-                                }
                                 goto wind;
                         }
 
@@ -645,10 +737,6 @@ afr_unlock_inodelk (call_frame_t *frame, xlator_t *this)
                         }
                         UNLOCK (&local->fd->lock);
 
-                        if (!fd_lock_owner) {
-                                afr_set_lk_owner (frame, this, local->fd);
-                                fd_lock_owner = _gf_true;
-                        }
                         if (piggyback) {
                                 afr_unlock_inodelk_cbk (frame, (void *) (long) i,
                                                         this, 1, 0, NULL);
@@ -668,7 +756,7 @@ afr_unlock_inodelk (call_frame_t *frame, xlator_t *this)
                                            (void *) (long)i,
                                            priv->children[i],
                                            priv->children[i]->fops->finodelk,
-                                           this->name, local->fd,
+                                           int_lock->domain, local->fd,
                                            F_SETLK, flock_use, NULL);
 
                         if (!--call_count)
@@ -683,7 +771,7 @@ afr_unlock_inodelk (call_frame_t *frame, xlator_t *this)
                                            (void *) (long)i,
                                            priv->children[i],
                                            priv->children[i]->fops->inodelk,
-                                           this->name, &local->loc,
+                                           int_lock->domain, &local->loc,
                                            F_SETLK, &flock, NULL);
 
                         if (!--call_count)
@@ -699,13 +787,22 @@ afr_unlock_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         afr_local_t         *local = NULL;
-        int32_t             child_index = (long)cookie;
+        afr_private_t       *priv  = NULL;
+        afr_internal_lock_t *int_lock = NULL;
+        int32_t             child_index = 0;
+        int                 lockee_no   = 0;
+
+        priv = this->private;
+        lockee_no = (int)((long) cookie) / priv->child_count;
+        child_index = (int) ((long) cookie) % priv->child_count;
 
         local = frame->local;
+        int_lock = &local->internal_lock;
 
         AFR_TRACE_ENTRYLK_OUT (frame, this, AFR_ENTRYLK_TRANSACTION,
-                               AFR_UNLOCK_OP, NULL, op_ret,
-                               op_errno, child_index);
+                               AFR_UNLOCK_OP,
+                               int_lock->lockee[lockee_no].basename, op_ret,
+                               op_errno, (int) ((long)cookie));
 
         if (op_ret < 0) {
                 gf_log (this->name, GF_LOG_ERROR,
@@ -713,6 +810,7 @@ afr_unlock_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->loc.path, child_index, strerror (op_errno));
         }
 
+        int_lock->lockee[lockee_no].locked_nodes[child_index] &= LOCKED_NO;
         afr_unlock_common_cbk (frame, cookie, this, op_ret, op_errno, NULL);
 
         return 0;
@@ -721,24 +819,22 @@ afr_unlock_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 static int
 afr_unlock_entrylk (call_frame_t *frame, xlator_t *this)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_private_t       *priv     = NULL;
-        const char          *basename = NULL;
-        loc_t               *loc      = NULL;
-        int call_count = 0;
-        int i = -1;
+        afr_internal_lock_t     *int_lock       = NULL;
+        afr_local_t             *local          = NULL;
+        afr_private_t           *priv           = NULL;
+        int                     call_count      = 0;
+        int                     index           = 0;
+        int                     lockee_no       = 0;
+        int                     copies          = 0;
+        int                     i               = -1;
 
         local    = frame->local;
         int_lock = &local->internal_lock;
         priv     = this->private;
+        copies   = priv->child_count;
 
-        basename = int_lock->lk_basename;
-        if (int_lock->lk_loc)
-                loc = int_lock->lk_loc;
+        call_count = afr_lockee_locked_nodes_count (int_lock);
 
-        call_count = afr_locked_nodes_count (int_lock->entry_locked_nodes,
-                                             priv->child_count);
         int_lock->lk_call_count = call_count;
 
         if (!call_count){
@@ -748,18 +844,22 @@ afr_unlock_entrylk (call_frame_t *frame, xlator_t *this)
                 goto out;
         }
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (int_lock->entry_locked_nodes[i] & LOCKED_YES) {
-                        AFR_TRACE_ENTRYLK_IN (frame, this,
-                                              AFR_ENTRYLK_NB_TRANSACTION,
-                                              AFR_UNLOCK_OP, basename, i);
+        for (i = 0; i < int_lock->lockee_count * priv->child_count; i++) {
+                lockee_no = i / copies;
+                index     = i % copies;
+                if (int_lock->lockee[lockee_no].locked_nodes[index] & LOCKED_YES) {
+                        AFR_TRACE_ENTRYLK_IN (frame, this, AFR_ENTRYLK_NB_TRANSACTION,
+                                              AFR_UNLOCK_OP,
+                                              int_lock->lockee[lockee_no].basename,
+                                              i);
 
                         STACK_WIND_COOKIE (frame, afr_unlock_entrylk_cbk,
                                            (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->entrylk,
-                                           this->name,
-                                           loc, basename,
+                                           priv->children[index],
+                                           priv->children[index]->fops->entrylk,
+                                           int_lock->domain,
+                                           &int_lock->lockee[lockee_no].loc,
+                                           int_lock->lockee[lockee_no].basename,
                                            ENTRYLK_UNLOCK, ENTRYLK_WRLCK, NULL);
 
                         if (!--call_count)
@@ -776,13 +876,20 @@ static int32_t
 afr_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
               int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        int child_index = (long) cookie;
+        afr_internal_lock_t     *int_lock       = NULL;
+        afr_local_t             *local          = NULL;
+        afr_private_t           *priv           = NULL;
+        int                     cky             = (long) cookie;
+        int                     child_index     = 0;
+        int                     lockee_no       = 0;
 
+        priv     = this->private;
         local    = frame->local;
         int_lock = &local->internal_lock;
 
+        child_index = ((int)cky) % priv->child_count;
+        lockee_no   = ((int)cky) / priv->child_count;
+
         LOCK (&frame->lock);
         {
                 if (op_ret == -1) {
@@ -798,6 +905,8 @@ afr_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->op_errno              = op_errno;
                         int_lock->lock_op_errno      = op_errno;
                 }
+
+		int_lock->lk_attempted_count++;
         }
         UNLOCK (&frame->lock);
 
@@ -806,10 +915,17 @@ afr_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 afr_unlock (frame, this);
         } else {
                 if (op_ret == 0) {
-                        int_lock->locked_nodes[child_index] |= LOCKED_YES;
-                        int_lock->lock_count++;
+                        if (local->transaction.type == AFR_ENTRY_TRANSACTION ||
+                            local->transaction.type == AFR_ENTRY_RENAME_TRANSACTION) {
+                                int_lock->lockee[lockee_no].locked_nodes[child_index] |= LOCKED_YES;
+                                int_lock->lockee[lockee_no].locked_count++;
+                                int_lock->entrylk_lock_count++;
+                        } else {
+                                int_lock->locked_nodes[child_index] |= LOCKED_YES;
+                                int_lock->lock_count++;
+                        }
                 }
-                afr_lock_blocking (frame, this, child_index + 1);
+                afr_lock_blocking (frame, this, cky + 1);
         }
 
         return 0;
@@ -829,79 +945,6 @@ afr_blocking_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 }
 
 static int32_t
-afr_lock_lower_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        afr_internal_lock_t *int_lock    = NULL;
-        afr_private_t       *priv        = NULL;
-        afr_local_t         *local       = NULL;
-        loc_t               *lower       = NULL;
-        loc_t               *higher      = NULL;
-        const char          *higher_name = NULL;
-        int child_index = (long) cookie;
-
-        priv     = this->private;
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        if (op_errno == ENOSYS) {
-                                /* return ENOTSUP */
-
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "subvolume does not support locking. "
-                                        "please load features/locks xlator on server");
-
-                                local->op_ret   = op_ret;
-                        }
-
-                        local->op_errno = op_errno;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if (op_ret != 0) {
-                //No need to lock higher name, move on to next one
-                afr_lock_blocking (frame, this, child_index + 1);
-                goto out;
-        } else {
-                int_lock->lower_locked_nodes[child_index] |= LOCKED_LOWER;
-                int_lock->lock_count++;
-        }
-
-        /* The lower path has been locked. Now lock the higher path */
-
-        lower = lower_path (&local->transaction.parent_loc,
-                            local->transaction.basename,
-                            &local->transaction.new_parent_loc,
-                            local->transaction.new_basename);
-
-        higher = (lower == &local->transaction.parent_loc ?
-                  &local->transaction.new_parent_loc :
-                  &local->transaction.parent_loc);
-
-        higher_name = (higher == &local->transaction.parent_loc ?
-                       local->transaction.basename :
-                       local->transaction.new_basename);
-
-        AFR_TRACE_ENTRYLK_IN (frame, this, AFR_ENTRYLK_TRANSACTION,
-                              AFR_LOCK_OP, higher_name, child_index);
-
-
-        STACK_WIND_COOKIE (frame, afr_lock_cbk,
-                           (void *) (long) child_index,
-                           priv->children[child_index],
-                           priv->children[child_index]->fops->entrylk,
-                           this->name, higher, higher_name,
-                           ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL);
-
-out:
-        return 0;
-}
-
-static int32_t
 afr_blocking_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                           int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
@@ -917,6 +960,7 @@ static int
 afr_copy_locked_nodes (call_frame_t *frame, xlator_t *this)
 {
         afr_internal_lock_t *int_lock = NULL;
+        afr_inodelk_t       *inodelk  = NULL;
         afr_local_t         *local    = NULL;
         afr_private_t       *priv     = NULL;
 
@@ -927,18 +971,16 @@ afr_copy_locked_nodes (call_frame_t *frame, xlator_t *this)
         switch (local->transaction.type) {
         case AFR_DATA_TRANSACTION:
         case AFR_METADATA_TRANSACTION:
-                memcpy (int_lock->inode_locked_nodes,
-                        int_lock->locked_nodes,
-                        priv->child_count);
-                int_lock->inodelk_lock_count = int_lock->lock_count;
+                inodelk = afr_get_inodelk (int_lock, int_lock->domain);
+                memcpy (inodelk->locked_nodes, int_lock->locked_nodes,
+                        sizeof (*inodelk->locked_nodes) * priv->child_count);
+                inodelk->lock_count = int_lock->lock_count;
                 break;
 
         case AFR_ENTRY_RENAME_TRANSACTION:
         case AFR_ENTRY_TRANSACTION:
-                memcpy (int_lock->entry_locked_nodes,
-                        int_lock->locked_nodes,
-                        priv->child_count);
-                int_lock->entrylk_lock_count = int_lock->lock_count;
+                /*entrylk_count is being used in both non-blocking and blocking
+                 * modes */
                 break;
         }
 
@@ -946,57 +988,67 @@ afr_copy_locked_nodes (call_frame_t *frame, xlator_t *this)
 
 }
 
-static gf_boolean_t
-did_blocking_lock_fail (call_frame_t *frame, xlator_t *this)
+static inline gf_boolean_t
+afr_is_entrylk (afr_internal_lock_t *int_lock,
+                afr_transaction_type trans_type)
 {
-        int                 i         = 0;
-        afr_local_t         *local    = NULL;
-        afr_internal_lock_t *int_lock = NULL;
-        afr_private_t       *priv     = NULL;
-        unsigned char       *lower    = NULL;
-        unsigned char       *higher   = NULL;
+        gf_boolean_t is_entrylk = _gf_false;
 
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        priv     = this->private;
+        if ((int_lock->transaction_lk_type == AFR_SELFHEAL_LK) &&
+            int_lock->selfheal_lk_type == AFR_ENTRY_SELF_HEAL_LK) {
 
-        if (local->transaction.type != AFR_ENTRY_RENAME_TRANSACTION) {
-                if (int_lock->lock_count == 0)
-                        return _gf_true;
-                else
-                        return _gf_false;
-        }
+                is_entrylk = _gf_true;
 
-        //Rename lock is successful when at least on one subvolume both lower
-        //and higher locks are acquired.
-        lower = int_lock->lower_locked_nodes;
-        higher = int_lock->locked_nodes;
-        for (i = 0; i < priv->child_count; i++) {
-                if ((lower[i] & LOCKED_LOWER) && (higher[i] & LOCKED_YES))
-                        return _gf_false;
+        } else if ((int_lock->transaction_lk_type == AFR_TRANSACTION_LK) &&
+                 (trans_type == AFR_ENTRY_TRANSACTION ||
+                  trans_type == AFR_ENTRY_RENAME_TRANSACTION)) {
+
+                is_entrylk = _gf_true;
+
+        } else {
+                is_entrylk = _gf_false;
         }
+
+        return is_entrylk;
+}
+
+static gf_boolean_t
+_is_lock_wind_needed (afr_local_t *local, int child_index)
+{
+        if (!local->child_up[child_index])
+                return _gf_false;
+
         return _gf_true;
 }
 
 int
-afr_lock_blocking (call_frame_t *frame, xlator_t *this, int child_index)
+afr_lock_blocking (call_frame_t *frame, xlator_t *this, int cookie)
 {
         afr_internal_lock_t *int_lock    = NULL;
+        afr_inodelk_t       *inodelk     = NULL;
         afr_local_t         *local       = NULL;
         afr_private_t       *priv        = NULL;
-        loc_t               *lower       = NULL;
-        const char          *lower_name  = NULL;
         struct gf_flock flock = {0,};
         uint64_t ctx = 0;
         int ret = 0;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        priv     = this->private;
-
-        flock.l_start = int_lock->lk_flock.l_start;
-        flock.l_len   = int_lock->lk_flock.l_len;
-        flock.l_type  = int_lock->lk_flock.l_type;
+        int child_index = 0;
+        int lockee_no   = 0;
+        gf_boolean_t is_entrylk = _gf_false;
+
+        local         = frame->local;
+        int_lock      = &local->internal_lock;
+        priv          = this->private;
+        child_index   = cookie % priv->child_count;
+        lockee_no     = cookie / priv->child_count;
+        is_entrylk    = afr_is_entrylk (int_lock, local->transaction.type);
+
+
+        if (!is_entrylk) {
+                inodelk = afr_get_inodelk (int_lock, int_lock->domain);
+                flock.l_start = inodelk->flock.l_start;
+                flock.l_len   = inodelk->flock.l_len;
+                flock.l_type  = inodelk->flock.l_type;
+        }
 
         if (local->fd) {
                 ret = fd_ctx_get (local->fd, this, &ctx);
@@ -1015,42 +1067,26 @@ afr_lock_blocking (call_frame_t *frame, xlator_t *this, int child_index)
 
                         return 0;
                 }
-
-                /* skip over children that or down
-                   or don't have the fd open */
-
-                while ((child_index < priv->child_count)
-                       && (!local->child_up[child_index] ||
-                          !local->fd_open_on[child_index]))
-
-                        child_index++;
-        } else {
-                /* skip over children that are down */
-                while ((child_index < priv->child_count)
-                       && !local->child_up[child_index])
-                        child_index++;
         }
 
-        if ((child_index == priv->child_count) &&
-            did_blocking_lock_fail (frame, this)) {
-
-                gf_log (this->name, GF_LOG_INFO,
-                        "unable to lock on even one child");
-
-                local->op_ret           = -1;
-                int_lock->lock_op_ret   = -1;
+        if (int_lock->lk_expected_count == int_lock->lk_attempted_count) {
+                if ((is_entrylk && int_lock->entrylk_lock_count == 0) ||
+                    (!is_entrylk && int_lock->lock_count == 0)) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "unable to lock on even one child");
 
-                afr_copy_locked_nodes (frame, this);
+                        local->op_ret           = -1;
+                        int_lock->lock_op_ret   = -1;
 
-                afr_unlock(frame, this);
+                        afr_copy_locked_nodes (frame, this);
 
-                return 0;
+                        afr_unlock(frame, this);
 
+                        return 0;
+                }
         }
 
-        if ((child_index == priv->child_count)
-            || (int_lock->lock_count == int_lock->lk_expected_count)) {
-
+        if (int_lock->lk_expected_count == int_lock->lk_attempted_count) {
                 /* we're done locking */
 
                 gf_log (this->name, GF_LOG_DEBUG,
@@ -1063,6 +1099,11 @@ afr_lock_blocking (call_frame_t *frame, xlator_t *this, int child_index)
                 return 0;
         }
 
+        if (!_is_lock_wind_needed (local, child_index)) {
+                afr_lock_blocking (frame, this, cookie + 1);
+                return 0;
+        }
+
         switch (local->transaction.type) {
         case AFR_DATA_TRANSACTION:
         case AFR_METADATA_TRANSACTION:
@@ -1077,7 +1118,7 @@ afr_lock_blocking (call_frame_t *frame, xlator_t *this, int child_index)
                                            (void *) (long) child_index,
                                            priv->children[child_index],
                                            priv->children[child_index]->fops->finodelk,
-                                           this->name, local->fd,
+                                           int_lock->domain, local->fd,
                                            F_SETLKW, &flock, NULL);
 
                 } else {
@@ -1090,50 +1131,29 @@ afr_lock_blocking (call_frame_t *frame, xlator_t *this, int child_index)
                                            (void *) (long) child_index,
                                            priv->children[child_index],
                                            priv->children[child_index]->fops->inodelk,
-                                           this->name, &local->loc,
+                                           int_lock->domain, &local->loc,
                                            F_SETLKW, &flock, NULL);
                 }
 
                 break;
 
         case AFR_ENTRY_RENAME_TRANSACTION:
-        {
-                lower = lower_path (&local->transaction.parent_loc,
-                                    local->transaction.basename,
-                                    &local->transaction.new_parent_loc,
-                                    local->transaction.new_basename);
-
-                lower_name = (lower == &local->transaction.parent_loc ?
-                              local->transaction.basename :
-                              local->transaction.new_basename);
-
-                AFR_TRACE_ENTRYLK_IN (frame, this, AFR_ENTRYLK_TRANSACTION,
-                                      AFR_LOCK_OP, lower_name, child_index);
-
-
-                STACK_WIND_COOKIE (frame, afr_lock_lower_cbk,
-                                   (void *) (long) child_index,
-                                   priv->children[child_index],
-                                   priv->children[child_index]->fops->entrylk,
-                                   this->name, lower, lower_name,
-                                   ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL);
-
-                break;
-        }
-
         case AFR_ENTRY_TRANSACTION:
+                /*Accounting for child_index increments on 'down'
+                 *and 'fd-less' children */
+
                 if (local->fd) {
-                        AFR_TRACE_ENTRYLK_IN (frame, this,
-                                              AFR_ENTRYLK_TRANSACTION,
-                                              AFR_LOCK_OP, local->transaction.basename,
-                                              child_index);
+                        AFR_TRACE_ENTRYLK_IN (frame, this, AFR_ENTRYLK_TRANSACTION,
+                                              AFR_LOCK_OP,
+                                              int_lock->lockee[lockee_no].basename,
+                                              cookie);
 
                         STACK_WIND_COOKIE (frame, afr_blocking_entrylk_cbk,
-                                           (void *) (long) child_index,
+                                           (void *) (long) cookie,
                                            priv->children[child_index],
                                            priv->children[child_index]->fops->fentrylk,
-                                           this->name, local->fd,
-                                           local->transaction.basename,
+                                           int_lock->domain, local->fd,
+                                           int_lock->lockee[lockee_no].basename,
                                            ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL);
                 } else {
                         AFR_TRACE_ENTRYLK_IN (frame, this,
@@ -1142,12 +1162,12 @@ afr_lock_blocking (call_frame_t *frame, xlator_t *this, int child_index)
                                               child_index);
 
                         STACK_WIND_COOKIE (frame, afr_blocking_entrylk_cbk,
-                                           (void *) (long) child_index,
+                                           (void *) (long) cookie,
                                            priv->children[child_index],
                                            priv->children[child_index]->fops->entrylk,
-                                           this->name,
-                                           &local->transaction.parent_loc,
-                                           local->transaction.basename,
+                                           int_lock->domain,
+                                           &int_lock->lockee[lockee_no].loc,
+                                           int_lock->lockee[lockee_no].basename,
                                            ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL);
                 }
 
@@ -1176,11 +1196,11 @@ afr_blocking_lock (call_frame_t *frame, xlator_t *this)
                 break;
 
         case AFR_ENTRY_RENAME_TRANSACTION:
-                up_count = afr_up_children_count (local->child_up,
-                                                  priv->child_count);
-                int_lock->lk_expected_count = 2 * up_count;
-                //fallthrough
         case AFR_ENTRY_TRANSACTION:
+                up_count = AFR_COUNT (local->child_up, priv->child_count);
+                int_lock->lk_call_count = int_lock->lk_expected_count
+                                        = (int_lock->lockee_count *
+                                           up_count);
                 initialize_entrylk_variables (frame, this);
                 break;
         }
@@ -1196,35 +1216,48 @@ afr_nonblocking_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 {
         afr_internal_lock_t *int_lock = NULL;
         afr_local_t         *local    = NULL;
-        int                 call_count  = 0;
-        int                 child_index = (long) cookie;
+        int call_count          = 0;
+        int child_index         = (long) cookie;
+        int copies              = 0;
+        int index               = 0;
+        int lockee_no           = 0;
+        afr_private_t       *priv = NULL;
+
+        priv = this->private;
+
+        copies = priv->child_count;
+        index = child_index % copies;
+        lockee_no = child_index / copies;
 
         local    = frame->local;
         int_lock = &local->internal_lock;
 
         AFR_TRACE_ENTRYLK_OUT (frame, this, AFR_ENTRYLK_TRANSACTION,
-                               AFR_LOCK_OP, NULL, op_ret,
+                               AFR_LOCK_OP,
+                               int_lock->lockee[lockee_no].basename, op_ret,
                                op_errno, (long) cookie);
 
-        if (op_ret < 0 ) {
-                if (op_errno == ENOSYS) {
+	LOCK (&frame->lock);
+	{
+		if (op_ret < 0 ) {
+			if (op_errno == ENOSYS) {
                         /* return ENOTSUP */
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "subvolume does not support locking. "
-                                "please load features/locks xlator on server");
-                        local->op_ret         = op_ret;
-                        int_lock->lock_op_ret = op_ret;
-
-                        int_lock->lock_op_errno      = op_errno;
-                        local->op_errno              = op_errno;
-                }
-        } else if (op_ret == 0) {
-                int_lock->entry_locked_nodes[child_index] |= LOCKED_YES;
-                int_lock->entrylk_lock_count++;
-        }
+				gf_log (this->name, GF_LOG_ERROR,
+					"subvolume does not support locking. "
+					"please load features/locks xlator on server");
+				local->op_ret         = op_ret;
+				int_lock->lock_op_ret = op_ret;
+
+				int_lock->lock_op_errno      = op_errno;
+				local->op_errno              = op_errno;
+			}
+		} else if (op_ret == 0) {
+			int_lock->lockee[lockee_no].locked_nodes[index] |= \
+				LOCKED_YES;
+			int_lock->lockee[lockee_no].locked_count++;
+			int_lock->entrylk_lock_count++;
+		}
 
-        LOCK (&frame->lock);
-        {
                 call_count = --int_lock->lk_call_count;
         }
         UNLOCK (&frame->lock);
@@ -1254,42 +1287,26 @@ afr_nonblocking_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         return 0;
 }
 
-void
-afr_mark_fd_open_on (afr_local_t *local, afr_fd_ctx_t *fd_ctx,
-                       size_t child_count)
-{
-        int             i = 0;
-
-        GF_ASSERT (local->fd_open_on);
-
-        memset (local->fd_open_on, 0, sizeof (*local->fd_open_on)*child_count);
-        for (i = 0; i < child_count; i++)
-                if (fd_ctx->opened_on[i] == AFR_FD_OPENED)
-                        local->fd_open_on[i] = 1;
-}
-
 int
 afr_nonblocking_entrylk (call_frame_t *frame, xlator_t *this)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_private_t       *priv     = NULL;
-        afr_fd_ctx_t        *fd_ctx   = NULL;
-        const char          *basename = NULL;
-        loc_t               *loc      = NULL;
-        int32_t call_count = 0;
+        afr_internal_lock_t *int_lock   = NULL;
+        afr_local_t         *local      = NULL;
+        afr_private_t       *priv       = NULL;
+        afr_fd_ctx_t        *fd_ctx     = NULL;
+        int                 copies      = 0;
+        int                 index       = 0;
+        int                 lockee_no   = 0;
+        int32_t             call_count  = 0;
         int i = 0;
 
         local    = frame->local;
         int_lock = &local->internal_lock;
         priv     = this->private;
 
+        copies = priv->child_count;
         initialize_entrylk_variables (frame, this);
 
-        basename = int_lock->lk_basename;
-        if (int_lock->lk_loc)
-                loc = int_lock->lk_loc;
-
         if (local->fd) {
                 fd_ctx = afr_fd_ctx_get (local->fd, this);
                 if (!fd_ctx) {
@@ -1302,11 +1319,11 @@ afr_nonblocking_entrylk (call_frame_t *frame, xlator_t *this)
                         local->op_errno         = EINVAL;
                         int_lock->lock_op_errno = EINVAL;
 
+			afr_unlock (frame, this);
                         return -1;
                 }
 
-                afr_mark_fd_open_on (local, fd_ctx, priv->child_count);
-                call_count = internal_lock_count (frame, this);
+                call_count = int_lock->lockee_count * internal_lock_count (frame, this);
                 int_lock->lk_call_count = call_count;
                 int_lock->lk_expected_count = call_count;
 
@@ -1319,46 +1336,52 @@ afr_nonblocking_entrylk (call_frame_t *frame, xlator_t *this)
 
                 /* Send non-blocking entrylk calls only on up children
                    and where the fd has been opened */
-                for (i = 0; i < priv->child_count; i++) {
-                        if (local->child_up[i] && local->fd_open_on[i]) {
-                                AFR_TRACE_ENTRYLK_IN (frame, this,
-                                                      AFR_ENTRYLK_NB_TRANSACTION,
-                                                      AFR_LOCK_OP, basename, i);
+                for (i = 0; i < int_lock->lockee_count*priv->child_count; i++) {
+                        index = i%copies;
+                        lockee_no = i/copies;
+                        if (local->child_up[index]) {
+                                AFR_TRACE_ENTRYLK_IN (frame, this, AFR_ENTRYLK_NB_TRANSACTION,
+                                                      AFR_LOCK_OP,
+                                                      int_lock->lockee[lockee_no].basename,
+                                                      i);
 
                                 STACK_WIND_COOKIE (frame, afr_nonblocking_entrylk_cbk,
                                                    (void *) (long) i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->fentrylk,
+                                                   priv->children[index],
+                                                   priv->children[index]->fops->fentrylk,
                                                    this->name, local->fd,
-                                                   basename,
+                                                   int_lock->lockee[lockee_no].basename,
                                                    ENTRYLK_LOCK_NB, ENTRYLK_WRLCK,
                                                    NULL);
+                                if (!--call_count)
+                                        break;
                         }
                 }
         } else {
-                GF_ASSERT (loc);
-
-                call_count = internal_lock_count (frame, this);
+                call_count = int_lock->lockee_count * internal_lock_count (frame, this);
                 int_lock->lk_call_count = call_count;
                 int_lock->lk_expected_count = call_count;
 
-                for (i = 0; i < priv->child_count; i++) {
-                        if (local->child_up[i]) {
-                                AFR_TRACE_ENTRYLK_IN (frame, this,
-                                                      AFR_ENTRYLK_NB_TRANSACTION,
-                                                      AFR_LOCK_OP, basename, i);
+                for (i = 0; i < int_lock->lockee_count*priv->child_count; i++) {
+                        index = i%copies;
+                        lockee_no = i/copies;
+                        if (local->child_up[index]) {
+                                AFR_TRACE_ENTRYLK_IN (frame, this, AFR_ENTRYLK_NB_TRANSACTION,
+                                                      AFR_LOCK_OP,
+                                                      int_lock->lockee[lockee_no].basename,
+                                                      i);
 
                                 STACK_WIND_COOKIE (frame, afr_nonblocking_entrylk_cbk,
                                                    (void *) (long) i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->entrylk,
-                                                   this->name, loc, basename,
+                                                   priv->children[index],
+                                                   priv->children[index]->fops->entrylk,
+                                                   this->name, &int_lock->lockee[lockee_no].loc,
+                                                   int_lock->lockee[lockee_no].basename,
                                                    ENTRYLK_LOCK_NB, ENTRYLK_WRLCK,
                                                    NULL);
 
                                 if (!--call_count)
                                         break;
-
                         }
                 }
         }
@@ -1371,6 +1394,7 @@ afr_nonblocking_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                              int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         afr_internal_lock_t *int_lock = NULL;
+        afr_inodelk_t       *inodelk  = NULL;
         afr_local_t         *local    = NULL;
         int call_count  = 0;
         int child_index = (long) cookie;
@@ -1379,58 +1403,57 @@ afr_nonblocking_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         local    = frame->local;
         int_lock = &local->internal_lock;
+        inodelk = afr_get_inodelk (int_lock, int_lock->domain);
 
         AFR_TRACE_INODELK_OUT (frame, this, AFR_INODELK_NB_TRANSACTION,
                                AFR_LOCK_OP, NULL, op_ret,
                                op_errno, (long) cookie);
 
-        if (op_ret < 0) {
-                if (op_errno == ENOSYS) {
-                        /* return ENOTSUP */
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "subvolume does not support locking. "
-                                "please load features/locks xlator on server");
-                        local->op_ret                = op_ret;
-                        int_lock->lock_op_ret        = op_ret;
-                        int_lock->lock_op_errno      = op_errno;
-                        local->op_errno              = op_errno;
-                }
-                if (local->transaction.eager_lock)
-                        local->transaction.eager_lock[child_index] = 0;
-        } else {
-                int_lock->inode_locked_nodes[child_index]
-                        |= LOCKED_YES;
-                int_lock->inodelk_lock_count++;
-
-                if (local->transaction.eager_lock &&
-                    local->transaction.eager_lock[child_index] && local->fd) {
-                        fd_ctx = afr_fd_ctx_get (local->fd, this);
-                        /* piggybacked */
-
-                        if (op_ret == 1) {
-                                /* piggybacked */
-                        } else if (op_ret == 0) {
-                                /* lock acquired from server */
-                                LOCK (&local->fd->lock);
-                                {
-                                        fd_ctx->lock_acquired[child_index]++;
-                                }
-                                UNLOCK (&local->fd->lock);
-                        }
-                }
-        }
+	if (local->fd)
+		fd_ctx = afr_fd_ctx_get (local->fd, this);
 
         LOCK (&frame->lock);
         {
+		if (op_ret < 0) {
+			if (op_errno == ENOSYS) {
+				/* return ENOTSUP */
+				gf_log (this->name, GF_LOG_ERROR,
+					"subvolume does not support locking. "
+					"please load features/locks xlator on "
+					"server");
+				local->op_ret                = op_ret;
+				int_lock->lock_op_ret        = op_ret;
+				int_lock->lock_op_errno      = op_errno;
+				local->op_errno              = op_errno;
+			}
+			if (local->transaction.eager_lock)
+				local->transaction.eager_lock[child_index] = 0;
+		} else {
+			inodelk->locked_nodes[child_index] |= LOCKED_YES;
+			inodelk->lock_count++;
+
+			if (local->transaction.eager_lock &&
+			    local->transaction.eager_lock[child_index] &&
+			    local->fd) {
+				/* piggybacked */
+				if (op_ret == 1) {
+					/* piggybacked */
+				} else if (op_ret == 0) {
+					/* lock acquired from server */
+                                        fd_ctx->lock_acquired[child_index]++;
+				}
+			}
+		}
+
                 call_count = --int_lock->lk_call_count;
         }
         UNLOCK (&frame->lock);
+
         if (call_count == 0) {
                 gf_log (this->name, GF_LOG_TRACE,
                         "Last inode locking reply received");
                 /* all locks successful. Proceed to call FOP */
-                if (int_lock->inodelk_lock_count ==
-                                int_lock->lk_expected_count) {
+                if (inodelk->lock_count == int_lock->lk_expected_count) {
                         gf_log (this->name, GF_LOG_TRACE,
                                 "All servers locked. Calling the cbk");
                         int_lock->lock_op_ret = 0;
@@ -1454,6 +1477,7 @@ int
 afr_nonblocking_inodelk (call_frame_t *frame, xlator_t *this)
 {
         afr_internal_lock_t *int_lock = NULL;
+        afr_inodelk_t       *inodelk  = NULL;
         afr_local_t         *local    = NULL;
         afr_private_t       *priv     = NULL;
         afr_fd_ctx_t        *fd_ctx   = NULL;
@@ -1464,17 +1488,18 @@ afr_nonblocking_inodelk (call_frame_t *frame, xlator_t *this)
         struct              gf_flock full_flock = {0,};
         struct              gf_flock *flock_use = NULL;
         int                 piggyback = 0;
-        gf_boolean_t        fd_lock_owner = _gf_false;
 
         local    = frame->local;
         int_lock = &local->internal_lock;
         priv     = this->private;
 
-        flock.l_start = int_lock->lk_flock.l_start;
-        flock.l_len   = int_lock->lk_flock.l_len;
-        flock.l_type  = int_lock->lk_flock.l_type;
+        inodelk = afr_get_inodelk (int_lock, int_lock->domain);
+
+        flock.l_start = inodelk->flock.l_start;
+        flock.l_len   = inodelk->flock.l_len;
+        flock.l_type  = inodelk->flock.l_type;
 
-        full_flock.l_type = int_lock->lk_flock.l_type;
+        full_flock.l_type = inodelk->flock.l_type;
 
         initialize_inodelk_variables (frame, this);
 
@@ -1490,11 +1515,11 @@ afr_nonblocking_inodelk (call_frame_t *frame, xlator_t *this)
                         local->op_errno         = EINVAL;
                         int_lock->lock_op_errno = EINVAL;
 
+			afr_unlock (frame, this);
                         ret = -1;
                         goto out;
                 }
 
-                afr_mark_fd_open_on (local, fd_ctx, priv->child_count);
                 call_count = internal_lock_count (frame, this);
                 int_lock->lk_call_count = call_count;
                 int_lock->lk_expected_count = call_count;
@@ -1509,22 +1534,19 @@ afr_nonblocking_inodelk (call_frame_t *frame, xlator_t *this)
                 /* Send non-blocking inodelk calls only on up children
                    and where the fd has been opened */
                 for (i = 0; i < priv->child_count; i++) {
-                        if (!local->child_up[i] || !local->fd_open_on[i])
+                        if (!local->child_up[i])
                                 continue;
 
                         flock_use = &flock;
-                        if (!priv->eager_lock) {
-                                if (fd_lock_owner) {
-                                        afr_set_lk_owner (frame, this,
-                                                          frame->root);
-                                        fd_lock_owner = _gf_false;
-                                }
+                        if (!local->transaction.eager_lock_on) {
                                 goto wind;
                         }
 
                         piggyback = 0;
                         local->transaction.eager_lock[i] = 1;
 
+			afr_set_delayed_post_op (frame, this);
+
                         LOCK (&local->fd->lock);
                         {
                                 if (fd_ctx->lock_acquired[i]) {
@@ -1534,11 +1556,6 @@ afr_nonblocking_inodelk (call_frame_t *frame, xlator_t *this)
                         }
                         UNLOCK (&local->fd->lock);
 
-                        if (!fd_lock_owner) {
-                                afr_set_lk_owner (frame, this, local->fd);
-                                fd_lock_owner = _gf_true;
-                        }
-
                         if (piggyback) {
                                 /* (op_ret == 1) => indicate piggybacked lock */
                                 afr_nonblocking_inodelk_cbk (frame, (void *) (long) i,
@@ -1557,7 +1574,7 @@ afr_nonblocking_inodelk (call_frame_t *frame, xlator_t *this)
                                            (void *) (long) i,
                                            priv->children[i],
                                            priv->children[i]->fops->finodelk,
-                                           this->name, local->fd,
+                                           int_lock->domain, local->fd,
                                            F_SETLK, flock_use, NULL);
 
                         if (!--call_count)
@@ -1579,7 +1596,7 @@ afr_nonblocking_inodelk (call_frame_t *frame, xlator_t *this)
                                            (void *) (long) i,
                                            priv->children[i],
                                            priv->children[i]->fops->inodelk,
-                                           this->name, &local->loc,
+                                           int_lock->domain, &local->loc,
                                            F_SETLK, &flock, NULL);
 
                         if (!--call_count)
@@ -1590,201 +1607,6 @@ out:
         return ret;
 }
 
-static int
-__is_lower_locked (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_private_t       *priv     = NULL;
-        afr_local_t         *local    = NULL;
-        int count = 0;
-        int i     = 0;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        priv     = this->private;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (int_lock->lower_locked_nodes[i] & LOCKED_LOWER)
-                        count++;
-        }
-
-        return count;
-
-}
-
-static int
-__is_higher_locked (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_private_t       *priv     = NULL;
-        afr_local_t         *local    = NULL;
-        int count = 0;
-        int i = 0;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        priv     = this->private;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (int_lock->locked_nodes[i] & LOCKED_YES)
-                        count++;
-        }
-
-        return count;
-
-}
-
-static int
-afr_unlock_lower_entrylk (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_private_t       *priv     = NULL;
-        const char          *basename = NULL;
-        loc_t               *loc      = NULL;
-        int call_count = 0;
-        int i = -1;
-
-        local = frame->local;
-        int_lock = &local->internal_lock;
-        priv  = this->private;
-
-        basename = int_lock->lk_basename;
-        if (int_lock->lk_loc)
-                loc = int_lock->lk_loc;
-
-        call_count = __is_lower_locked (frame, this);
-        int_lock->lk_call_count = call_count;
-
-        if (!call_count){
-                gf_log (this->name, GF_LOG_TRACE,
-                        "No internal locks unlocked");
-                int_lock->lock_cbk (frame, this);
-                goto out;
-        }
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (int_lock->lower_locked_nodes[i] & LOCKED_LOWER) {
-                        AFR_TRACE_ENTRYLK_IN (frame, this,
-                                              AFR_ENTRYLK_NB_TRANSACTION,
-                                              AFR_UNLOCK_OP, basename, i);
-
-                        STACK_WIND_COOKIE (frame, afr_unlock_entrylk_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->entrylk,
-                                           this->name,
-                                           loc, basename,
-                                           ENTRYLK_UNLOCK, ENTRYLK_WRLCK, NULL);
-
-                        if (!--call_count)
-                                break;
-
-                }
-        }
-
-out:
-        return 0;
-
-}
-
-
-static int
-afr_post_unlock_higher_cbk (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local = NULL;
-
-        local = frame->local;
-
-        local->transaction.done (frame, this);
-        return 0;
-}
-
-static int
-afr_post_unlock_lower_cbk (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock    = NULL;
-        afr_local_t         *local       = NULL;
-        loc_t               *lower       = NULL;
-        loc_t               *higher      = NULL;
-        const char          *higher_name = NULL;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        lower = lower_path (&local->transaction.parent_loc,
-                            local->transaction.basename,
-                            &local->transaction.new_parent_loc,
-                            local->transaction.new_basename);
-
-        higher = (lower == &local->transaction.parent_loc ?
-                  &local->transaction.new_parent_loc :
-                  &local->transaction.parent_loc);
-
-        higher_name = (higher == &local->transaction.parent_loc ?
-                       local->transaction.basename :
-                       local->transaction.new_basename);
-
-        if (__is_higher_locked (frame, this)) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "unlocking higher");
-                int_lock->lk_basename = higher_name;
-                int_lock->lk_loc      = higher;
-                int_lock->lock_cbk    = afr_post_unlock_higher_cbk;
-
-                afr_unlock_entrylk (frame, this);
-        } else
-                local->transaction.done (frame, this);
-
-        return 0;
-}
-
-static int
-afr_rename_unlock (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock    = NULL;
-        afr_local_t         *local       = NULL;
-        loc_t               *lower       = NULL;
-        const char          *lower_name  = NULL;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        lower = lower_path (&local->transaction.parent_loc,
-                            local->transaction.basename,
-                            &local->transaction.new_parent_loc,
-                            local->transaction.new_basename);
-
-        lower_name = (lower == &local->transaction.parent_loc ?
-                      local->transaction.basename :
-                      local->transaction.new_basename);
-
-        if (__is_lower_locked (frame, this)) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "unlocking lower");
-                int_lock->lk_basename = lower_name;
-                int_lock->lk_loc      = lower;
-                int_lock->lock_cbk    = afr_post_unlock_lower_cbk;
-
-                afr_unlock_lower_entrylk (frame, this);
-        } else
-                afr_post_unlock_lower_cbk (frame, this);
-
-        return 0;
-}
-
-static int
-afr_rename_transaction (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local = NULL;
-
-        local = frame->local;
-
-        return (local->transaction.type ==
-                AFR_ENTRY_RENAME_TRANSACTION);
-
-}
-
 int32_t
 afr_unlock (call_frame_t *frame, xlator_t *this)
 {
@@ -1793,14 +1615,11 @@ afr_unlock (call_frame_t *frame, xlator_t *this)
         local = frame->local;
 
         if (transaction_lk_op (local)) {
-                afr_set_lk_owner (frame, this, frame->root);
                 if (is_afr_lock_transaction (local))
                         afr_unlock_inodelk (frame, this);
                 else
-                        if (!afr_rename_transaction (frame, this))
-                                afr_unlock_entrylk (frame, this);
-                        else
-                                afr_rename_unlock (frame, this);
+                        afr_unlock_entrylk (frame, this);
+
         } else {
                 if (is_afr_lock_selfheal (local))
                         afr_unlock_inodelk (frame, this);
@@ -1812,518 +1631,37 @@ afr_unlock (call_frame_t *frame, xlator_t *this)
 }
 
 int
-afr_mark_locked_nodes (xlator_t *this, fd_t *fd,
-                       unsigned char *locked_nodes)
-{
-        afr_private_t *priv  = NULL;
-        afr_fd_ctx_t  *fdctx = NULL;
-        uint64_t       tmp   = 0;
-        int            ret   = 0;
-
-        priv = this->private;
-
-        ret = afr_fd_ctx_set (this, fd);
-        if (ret)
-                goto out;
-
-        ret = fd_ctx_get (fd, this, &tmp);
-        if (ret) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "failed to get the fd ctx");
-                goto out;
-        }
-        fdctx = (afr_fd_ctx_t *) (long) tmp;
-
-        GF_ASSERT (fdctx->locked_on);
-
-        memcpy (fdctx->locked_on, locked_nodes,
-                priv->child_count);
-
-out:
-        return ret;
-}
-
-static int
-__is_fd_saved (xlator_t *this, fd_t *fd)
-{
-        afr_locked_fd_t *locked_fd = NULL;
-        afr_private_t   *priv      = NULL;
-        int              found     = 0;
-
-        priv = this->private;
-
-        list_for_each_entry (locked_fd, &priv->saved_fds, list) {
-                if (locked_fd->fd == fd) {
-                        found = 1;
-                        break;
-                }
-        }
-
-        return found;
-}
-
-static int
-__afr_save_locked_fd (xlator_t *this, fd_t *fd)
-{
-        afr_private_t   *priv      = NULL;
-        afr_locked_fd_t *locked_fd = NULL;
-        int              ret       = 0;
-
-        priv = this->private;
-
-        locked_fd = GF_CALLOC (1, sizeof (*locked_fd),
-                               gf_afr_mt_locked_fd);
-        if (!locked_fd) {
-                ret = -1;
-                goto out;
-        }
-
-        locked_fd->fd = fd;
-        INIT_LIST_HEAD (&locked_fd->list);
-
-        list_add_tail (&locked_fd->list, &priv->saved_fds);
-
-out:
-        return ret;
-}
-
-int
-afr_save_locked_fd (xlator_t *this, fd_t *fd)
-{
-        afr_private_t   *priv      = NULL;
-        int              ret       = 0;
-
-        priv = this->private;
-
-        pthread_mutex_lock (&priv->mutex);
-        {
-                if (__is_fd_saved (this, fd)) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "fd=%p already saved", fd);
-                        goto unlock;
-                }
-
-                ret = __afr_save_locked_fd (this, fd);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "fd=%p could not be saved", fd);
-                        goto unlock;
-                }
-        }
-unlock:
-        pthread_mutex_unlock (&priv->mutex);
-
-        return ret;
-}
-
-static int
-afr_lock_recovery_cleanup (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t     *local     = NULL;
-        afr_locked_fd_t *locked_fd = NULL;
-
-        local = frame->local;
-
-        locked_fd = local->locked_fd;
-
-        STACK_DESTROY (frame->root);
-        afr_local_cleanup (local, this);
-
-        afr_save_locked_fd (this, locked_fd->fd);
-
-        return 0;
-
-}
-
-static int
-afr_get_source_lock_recovery (xlator_t *this, fd_t *fd)
-{
-        afr_fd_ctx_t  *fdctx        = NULL;
-        afr_private_t *priv         = NULL;
-        uint64_t      tmp           = 0;
-        int           i             = 0;
-        int           source_child  = -1;
-        int           ret           = 0;
-
-        priv = this->private;
-
-        ret = fd_ctx_get (fd, this, &tmp);
-        if (ret)
-                goto out;
-
-        fdctx = (afr_fd_ctx_t *) (long) tmp;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (fdctx->locked_on[i]) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "Found lock recovery source=%d", i);
-                        source_child = i;
-                        break;
-                }
-        }
-
-out:
-        return source_child;
-
-}
-
-int32_t
-afr_get_locks_fd_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
-                      dict_t *xdata);
-int32_t
-afr_recover_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
-                      dict_t *xdata)
-{
-        afr_local_t   *local = NULL;
-        afr_private_t *priv  = NULL;
-        int32_t source_child = 0;
-        struct gf_flock flock   = {0,};
-
-        local = frame->local;
-        priv  = this->private;
-
-        if (op_ret) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "lock recovery failed");
-                goto cleanup;
-        }
-
-        source_child = local->source_child;
-
-        memcpy (&flock, lock, sizeof (*lock));
-
-        STACK_WIND_COOKIE (frame, afr_get_locks_fd_cbk,
-                           (void *) (long) source_child,
-                           priv->children[source_child],
-                           priv->children[source_child]->fops->lk,
-                           local->fd, F_GETLK_FD, &flock, NULL);
-
-        return 0;
-
-cleanup:
-        afr_lock_recovery_cleanup (frame, this);
-        return 0;
-}
-
-int
-afr_recover_lock (call_frame_t *frame, xlator_t *this,
-                  struct gf_flock *flock)
-{
-        afr_local_t   *local             = NULL;
-        afr_private_t *priv              = NULL;
-        int32_t      lock_recovery_child = 0;
-
-        priv  = this->private;
-        local = frame->local;
-
-        lock_recovery_child = local->lock_recovery_child;
-
-        frame->root->lk_owner = flock->l_owner;
-
-        STACK_WIND_COOKIE (frame, afr_recover_lock_cbk,
-                           (void *) (long) lock_recovery_child,
-                           priv->children[lock_recovery_child],
-                           priv->children[lock_recovery_child]->fops->lk,
-                           local->fd, F_SETLK, flock, NULL);
-
-        return 0;
-}
-
-static int
-is_afr_lock_eol (struct gf_flock *lock)
-{
-        int ret = 0;
-
-        if ((lock->l_type == GF_LK_EOL))
-                ret = 1;
-
-        return ret;
-}
-
-int32_t
-afr_get_locks_fd_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
-                      dict_t *xdata)
-{
-        if (op_ret) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "Failed to get locks on fd");
-                goto cleanup;
-        }
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Got a lock on fd");
-
-        if (is_afr_lock_eol (lock)) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "Reached EOL on locks on fd");
-                goto cleanup;
-        }
-
-        afr_recover_lock (frame, this, lock);
-
-        return 0;
-
-cleanup:
-        afr_lock_recovery_cleanup (frame, this);
-
-        return 0;
-}
-
-static int
-afr_lock_recovery (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t   *local        = NULL;
-        afr_private_t *priv         = NULL;
-        fd_t          *fd           = NULL;
-        int            ret          = 0;
-        int32_t        source_child = 0;
-        struct gf_flock   flock        = {0,};
-
-        priv  = this->private;
-        local = frame->local;
-
-        fd = local->fd;
-
-        source_child = afr_get_source_lock_recovery (this, fd);
-        if (source_child < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Could not recover locks due to lock "
-                        "split brain");
-                ret = -1;
-                goto out;
-        }
-
-        local->source_child = source_child;
-
-        /* the flock can be zero filled as we're querying incrementally
-           the locks held on the fd.
-        */
-        STACK_WIND_COOKIE (frame, afr_get_locks_fd_cbk,
-                           (void *) (long) source_child,
-                           priv->children[source_child],
-                           priv->children[source_child]->fops->lk,
-                           local->fd, F_GETLK_FD, &flock, NULL);
-
-out:
-        return ret;
-}
-
-
-static int
-afr_mark_fd_opened (xlator_t *this, fd_t *fd, int32_t child_index)
-{
-        afr_fd_ctx_t *fdctx = NULL;
-        uint64_t      tmp   = 0;
-        int           ret   = 0;
-
-        ret = fd_ctx_get (fd, this, &tmp);
-        if (ret)
-                goto out;
-
-        fdctx = (afr_fd_ctx_t *) (long) tmp;
-
-        fdctx->opened_on[child_index] = AFR_FD_OPENED;
-
-out:
-        return ret;
-}
-
-int32_t
-afr_lock_recovery_preopen_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                               int32_t op_ret, int32_t op_errno, fd_t *fd,
-                               dict_t *xdata)
-{
-        int32_t child_index = (long )cookie;
-        int ret = 0;
-
-        if (op_ret) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "Reopen during lock-recovery failed");
-                goto cleanup;
-        }
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Open succeeded => proceed to recover locks");
-
-        ret = afr_lock_recovery (frame, this);
-        if (ret) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "Lock recovery failed");
-                goto cleanup;
-        }
-
-        ret = afr_mark_fd_opened (this, fd, child_index);
-        if (ret) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "Marking fd open failed");
-                goto cleanup;
-        }
-
-        return 0;
-
-cleanup:
-        afr_lock_recovery_cleanup (frame, this);
-        return 0;
-}
-
-static int
-afr_lock_recovery_preopen (call_frame_t *frame, xlator_t *this)
-{
-        afr_private_t *priv  = NULL;
-        afr_local_t   *local = NULL;
-        uint64_t       tmp   = 0;
-        afr_fd_ctx_t  *fdctx = NULL;
-        loc_t          loc   = {0,};
-        int32_t        child_index = 0;
-        int            ret   = 0;
-
-        priv  = this->private;
-        local = frame->local;
-
-        GF_ASSERT (local && local->fd);
-
-        ret = fd_ctx_get (local->fd, this, &tmp);
-        if (ret)
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to get the context of fd",
-                        uuid_utoa (local->fd->inode->gfid));
-        fdctx = (afr_fd_ctx_t *) (long) tmp;
-        /* TODO: instead we should return from the function */
-        GF_ASSERT (fdctx);
-
-        child_index = local->lock_recovery_child;
-
-        inode_path (local->fd->inode, NULL, (char **)&loc.path);
-        loc.name   = strrchr (loc.path, '/');
-        loc.inode  = inode_ref (local->fd->inode);
-        loc.parent = inode_parent (local->fd->inode, 0, NULL);
-
-
-        STACK_WIND_COOKIE (frame, afr_lock_recovery_preopen_cbk,
-                           (void *)(long) child_index,
-                           priv->children[child_index],
-                           priv->children[child_index]->fops->open,
-                           &loc, fdctx->flags, local->fd, NULL);
-
-        return 0;
-}
-
-static int
-is_fd_opened (fd_t *fd, int32_t child_index)
-{
-        afr_fd_ctx_t *fdctx = NULL;
-        uint64_t      tmp = 0;
-        int           ret = 0;
-
-        ret = fd_ctx_get (fd, THIS, &tmp);
-        if (ret)
-                goto out;
-
-        fdctx = (afr_fd_ctx_t *) (long) tmp;
-
-        if (fdctx->opened_on[child_index] == AFR_FD_OPENED)
-                ret = 1;
-
-out:
-        return ret;
-}
-
-int
-afr_attempt_lock_recovery (xlator_t *this, int32_t child_index)
-{
-        call_frame_t    *frame      = NULL;
-        afr_private_t   *priv       = NULL;
-        afr_local_t     *local      = NULL;
-        afr_locked_fd_t *locked_fd  = NULL;
-        afr_locked_fd_t  *tmp       = NULL;
-        int              ret        = -1;
-        struct list_head locks_list = {0,};
-        int32_t          op_errno   = 0;
-
-
-        priv = this->private;
-
-        if (list_empty (&priv->saved_fds))
-                goto out;
-
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame) {
-                ret = -1;
-                goto out;
-        }
-
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0) {
-                ret = -1;
-                goto out;
-        }
-
-        frame->local = local;
-
-        INIT_LIST_HEAD (&locks_list);
-
-        pthread_mutex_lock (&priv->mutex);
-        {
-                list_splice_init (&priv->saved_fds, &locks_list);
-        }
-        pthread_mutex_unlock (&priv->mutex);
-
-        list_for_each_entry_safe (locked_fd, tmp,
-                                  &locks_list, list) {
-
-                list_del_init (&locked_fd->list);
-
-                local->fd                  = fd_ref (locked_fd->fd);
-                local->lock_recovery_child = child_index;
-                local->locked_fd           = locked_fd;
-
-                if (!is_fd_opened (locked_fd->fd, child_index)) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "attempting open before lock "
-                                "recovery");
-                        afr_lock_recovery_preopen (frame, this);
-                } else {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "attempting lock recovery "
-                                "without a preopen");
-                        afr_lock_recovery (frame, this);
-                }
-        }
-
-out:
-        if ((ret < 0) && frame)
-                AFR_STACK_DESTROY (frame);
-        return ret;
-}
-
-void
-afr_lk_transfer_datalock (call_frame_t *dst, call_frame_t *src,
+afr_lk_transfer_datalock (call_frame_t *dst, call_frame_t *src, char *dom,
                           unsigned int child_count)
 {
-        afr_local_t *dst_local = NULL;
-        afr_local_t *src_local = NULL;
-        afr_internal_lock_t *dst_lock = NULL;
-        afr_internal_lock_t *src_lock = NULL;
+        afr_local_t         *dst_local   = NULL;
+        afr_local_t         *src_local   = NULL;
+        afr_internal_lock_t *dst_lock    = NULL;
+        afr_internal_lock_t *src_lock    = NULL;
+        afr_inodelk_t       *dst_inodelk = NULL;
+        afr_inodelk_t       *src_inodelk = NULL;
+        int                 ret = -1;
 
-        dst_local = dst->local;
-        dst_lock  = &dst_local->internal_lock;
         src_local = src->local;
         src_lock  = &src_local->internal_lock;
-        if (src_lock->inode_locked_nodes) {
-                memcpy (dst_lock->inode_locked_nodes,
-                        src_lock->inode_locked_nodes,
-                        sizeof (*dst_lock->inode_locked_nodes) * child_count);
-                memset (src_lock->inode_locked_nodes, 0,
-                        sizeof (*src_lock->inode_locked_nodes) * child_count);
+        src_inodelk = afr_get_inodelk (src_lock, dom);
+        dst_local = dst->local;
+        dst_lock  = &dst_local->internal_lock;
+        dst_inodelk = afr_get_inodelk (dst_lock, dom);
+        if (!dst_inodelk || !src_inodelk)
+                goto out;
+        if (src_inodelk->locked_nodes) {
+                memcpy (dst_inodelk->locked_nodes, src_inodelk->locked_nodes,
+                        sizeof (*dst_inodelk->locked_nodes) * child_count);
+                memset (src_inodelk->locked_nodes, 0,
+                        sizeof (*src_inodelk->locked_nodes) * child_count);
         }
 
         dst_lock->transaction_lk_type = src_lock->transaction_lk_type;
         dst_lock->selfheal_lk_type    = src_lock->selfheal_lk_type;
-        dst_lock->inodelk_lock_count = src_lock->inodelk_lock_count;
-        src_lock->inodelk_lock_count = 0;
+        dst_inodelk->lock_count = src_inodelk->lock_count;
+        src_inodelk->lock_count = 0;
+        ret = 0;
+out:
+        return ret;
 }
diff --git a/xlators/cluster/afr/src/afr-mem-types.h b/xlators/cluster/afr/src/afr-mem-types.h
index 7e684801f..05df90cc0 100644
--- a/xlators/cluster/afr/src/afr-mem-types.h
+++ b/xlators/cluster/afr/src/afr-mem-types.h
@@ -41,6 +41,8 @@ enum gf_afr_mem_types_ {
         gf_afr_mt_shd_event_t,
         gf_afr_mt_time_t,
         gf_afr_mt_pos_data_t,
+	gf_afr_mt_reply_t,
+	gf_afr_mt_subvol_healer_t,
         gf_afr_mt_end
 };
 #endif
diff --git a/xlators/cluster/afr/src/afr-open.c b/xlators/cluster/afr/src/afr-open.c
index c0be197f2..f86aa7fd8 100644
--- a/xlators/cluster/afr/src/afr-open.c
+++ b/xlators/cluster/afr/src/afr-open.c
@@ -43,85 +43,29 @@
 #include "afr-dir-read.h"
 #include "afr-dir-write.h"
 #include "afr-transaction.h"
-#include "afr-self-heal.h"
-#include "afr-self-heal-common.h"
 
-int
-afr_stale_child_up (afr_local_t *local, xlator_t *this)
-{
-        int             i = 0;
-        afr_private_t   *priv = NULL;
-        int             up = -1;
-
-        priv = this->private;
 
-        if (!local->fresh_children)
-                local->fresh_children = afr_children_create (priv->child_count);
-        if (!local->fresh_children)
-                goto out;
-
-        afr_inode_get_read_ctx (this, local->fd->inode, local->fresh_children);
-        if (priv->child_count == afr_get_children_count (local->fresh_children,
-                                                         priv->child_count))
-                goto out;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (!local->child_up[i])
-                        continue;
-                if (afr_is_child_present (local->fresh_children,
-                                          priv->child_count, i))
-                        continue;
-                up = i;
-                break;
-        }
-out:
-        return up;
-}
-
-void
-afr_perform_data_self_heal (call_frame_t *frame, xlator_t *this)
+gf_boolean_t
+afr_is_fd_fixable (fd_t *fd)
 {
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        inode_t         *inode = NULL;
-        int             st_child = -1;
-        char            reason[64] = {0};
-
-        local = frame->local;
-        sh = &local->self_heal;
-        inode = local->fd->inode;
-
-        if (!IA_ISREG (inode->ia_type))
-                goto out;
-
-        st_child = afr_stale_child_up (local, this);
-        if (st_child < 0)
-                goto out;
-
-        sh->do_data_self_heal          = _gf_true;
-        sh->do_metadata_self_heal      = _gf_true;
-        sh->do_gfid_self_heal          = _gf_true;
-        sh->do_missing_entry_self_heal = _gf_true;
-
-        snprintf (reason, sizeof (reason), "stale subvolume %d detected",
-                  st_child);
-        afr_launch_self_heal (frame, this, inode, _gf_true, inode->ia_type,
-                              reason, NULL, NULL);
-out:
-        return;
+        if (!fd || !fd->inode)
+                return _gf_false;
+        else if (fd_is_anonymous (fd))
+                return _gf_false;
+        else if (uuid_is_null (fd->inode->gfid))
+                return _gf_false;
+
+        return _gf_true;
 }
 
+
 int
 afr_open_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
                         struct iatt *postbuf, dict_t *xdata)
 {
         afr_local_t * local = frame->local;
-        afr_private_t *priv = NULL;
 
-        priv = this->private;
-        if (afr_open_only_data_self_heal (priv->data_self_heal))
-                afr_perform_data_self_heal (frame, this);
         AFR_STACK_UNWIND (open, frame, local->op_ret, local->op_errno,
                           local->fd, xdata);
         return 0;
@@ -134,49 +78,38 @@ afr_open_cbk (call_frame_t *frame, void *cookie,
               fd_t *fd, dict_t *xdata)
 {
         afr_local_t *  local       = NULL;
-        int            ret         = 0;
         int            call_count  = -1;
         int            child_index = (long) cookie;
-        afr_private_t *priv        = NULL;
+	afr_fd_ctx_t  *fd_ctx = NULL;
 
-        priv = this->private;
         local = frame->local;
+	fd_ctx = local->fd_ctx;
 
         LOCK (&frame->lock);
         {
                 if (op_ret == -1) {
                         local->op_errno = op_errno;
-                }
-
-                if (op_ret >= 0) {
+			fd_ctx->opened_on[child_index] = AFR_FD_NOT_OPENED;
+                } else {
                         local->op_ret = op_ret;
-                        local->success_count++;
-
-                        ret = afr_child_fd_ctx_set (this, fd, child_index,
-                                                    local->cont.open.flags);
-                        if (ret) {
-                                local->op_ret = -1;
-                                local->op_errno = -ret;
-                                goto unlock;
-                        }
+			fd_ctx->opened_on[child_index] = AFR_FD_OPENED;
+			if (!local->xdata_rsp && xdata)
+				local->xdata_rsp = dict_ref (xdata);
                 }
         }
-unlock:
         UNLOCK (&frame->lock);
 
         call_count = afr_frame_return (frame);
 
         if (call_count == 0) {
-                if ((local->cont.open.flags & O_TRUNC)
-                    && (local->op_ret >= 0)) {
+                if ((fd_ctx->flags & O_TRUNC) && (local->op_ret >= 0)) {
                         STACK_WIND (frame, afr_open_ftruncate_cbk,
                                     this, this->fops->ftruncate,
                                     fd, 0, NULL);
                 } else {
-                        if (afr_open_only_data_self_heal (priv->data_self_heal))
-                                afr_perform_data_self_heal (frame, this);
                         AFR_STACK_UNWIND (open, frame, local->op_ret,
-                                          local->op_errno, local->fd, xdata);
+                                          local->op_errno, local->fd,
+					  local->xdata_rsp);
                 }
         }
 
@@ -190,16 +123,11 @@ afr_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
         afr_private_t * priv       = NULL;
         afr_local_t *   local      = NULL;
         int             i          = 0;
-        int             ret        = -1;
         int32_t         call_count = 0;
         int32_t         op_errno   = 0;
-        int32_t         wind_flags = flags & (~O_TRUNC);
-        //We can't let truncation to happen outside transaction.
+	afr_fd_ctx_t   *fd_ctx = NULL;
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
-        VALIDATE_OR_GOTO (loc, out);
+        //We can't let truncation to happen outside transaction.
 
         priv = this->private;
 
@@ -207,240 +135,203 @@ afr_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
                 QUORUM_CHECK(open,out);
         }
 
-        if (afr_is_split_brain (this, loc->inode)) {
-                /* self-heal failed */
-                gf_log (this->name, GF_LOG_WARNING,
-                        "failed to open as split brain seen, returning EIO");
-                op_errno = EIO;
-                goto out;
-        }
-
-        AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-        local = frame->local;
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
 
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
+	fd_ctx = afr_fd_ctx_get (fd, this);
+	if (!fd_ctx) {
+		op_errno = ENOMEM;
+		goto out;
+	}
 
-        call_count   = local->call_count;
-        loc_copy (&local->loc, loc);
+        local->fd = fd_ref (fd);
+	local->fd_ctx = fd_ctx;
+	fd_ctx->flags = flags;
 
-        local->cont.open.flags   = flags;
+        call_count = local->call_count;
 
-        local->fd = fd_ref (fd);
+        local->cont.open.flags = flags;
 
         for (i = 0; i < priv->child_count; i++) {
                 if (local->child_up[i]) {
                         STACK_WIND_COOKIE (frame, afr_open_cbk, (void *) (long) i,
                                            priv->children[i],
                                            priv->children[i]->fops->open,
-                                           loc, wind_flags, fd, xdata);
-
+                                           loc, (flags & ~O_TRUNC), fd, xdata);
                         if (!--call_count)
                                 break;
                 }
         }
 
-        ret = 0;
+	return 0;
 out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (open, frame, -1, op_errno, fd, xdata);
+	AFR_STACK_UNWIND (open, frame, -1, op_errno, fd, NULL);
 
         return 0;
 }
 
-//NOTE: this function should be called with holding the lock on
-//fd to which fd_ctx belongs
-void
-afr_get_resumable_calls (xlator_t *this, afr_fd_ctx_t *fd_ctx,
-                         struct list_head *list)
-{
-        afr_fd_paused_call_t *paused_call = NULL;
-        afr_fd_paused_call_t *tmp = NULL;
-        afr_local_t           *call_local  = NULL;
-        afr_private_t         *priv        = NULL;
-        int                    i = 0;
-        gf_boolean_t           call = _gf_false;
-
-        priv = this->private;
-        list_for_each_entry_safe (paused_call, tmp, &fd_ctx->paused_calls,
-                                  call_list) {
-                call = _gf_true;
-                call_local = paused_call->frame->local;
-                for (i = 0; i < priv->child_count; i++) {
-                        if (call_local->child_up[i] &&
-                            (fd_ctx->opened_on[i] == AFR_FD_OPENING))
-                                call = _gf_false;
-                }
-
-                if (call) {
-                        list_del_init (&paused_call->call_list);
-                        list_add (&paused_call->call_list, list);
-                }
-        }
-}
-
-void
-afr_resume_calls (xlator_t *this, struct list_head *list)
-{
-        afr_fd_paused_call_t *paused_call = NULL;
-        afr_fd_paused_call_t *tmp = NULL;
-        afr_local_t           *call_local  = NULL;
-
-        list_for_each_entry_safe (paused_call, tmp, list, call_list) {
-                list_del_init (&paused_call->call_list);
-                call_local = paused_call->frame->local;
-                call_local->fop_call_continue (paused_call->frame, this);
-                GF_FREE (paused_call);
-        }
-}
-
 int
 afr_openfd_fix_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+                         int32_t op_ret, int32_t op_errno, fd_t *fd,
+                         dict_t *xdata)
 {
-        afr_local_t           *local       = NULL;
-        afr_private_t         *priv        = NULL;
-        afr_fd_ctx_t          *fd_ctx      = NULL;
-        int                    call_count  = 0;
-        int                    child_index = (long) cookie;
-        struct list_head       paused_calls = {0};
-        gf_boolean_t           fop_paused = _gf_false;
-        fd_t                   *local_fd  = NULL;
+        afr_local_t   *local      = NULL;
+        afr_private_t *priv       = NULL;
+        afr_fd_ctx_t  *fd_ctx     = NULL;
+        int           call_count  = 0;
+        int           child_index = (long) cookie;
 
         priv     = this->private;
         local    = frame->local;
-        fop_paused = local->fop_paused;
 
         if (op_ret >= 0) {
-                gf_log (this->name, GF_LOG_INFO, "fd for %s opened "
+                gf_log (this->name, GF_LOG_DEBUG, "fd for %s opened "
                         "successfully on subvolume %s", local->loc.path,
                         priv->children[child_index]->name);
+        } else {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to open %s "
+                        "on subvolume %s", local->loc.path,
+                        priv->children[child_index]->name);
         }
 
-        local_fd = fd_ref (local->fd);
-        call_count = afr_frame_return (frame);
-        //Note: Do not access any thing using the frame outside call_count 0
+        fd_ctx = local->fd_ctx;
 
-        //Note: No frame locking needed for this block of code
-        fd_ctx = afr_fd_ctx_get (local_fd, this);
-        if (!fd_ctx) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "failed to get fd context, %p", local_fd);
-                goto out;
-        }
-
-        LOCK (&local_fd->lock);
+        LOCK (&local->fd->lock);
         {
                 if (op_ret >= 0) {
                         fd_ctx->opened_on[child_index] = AFR_FD_OPENED;
                 } else {
                         fd_ctx->opened_on[child_index] = AFR_FD_NOT_OPENED;
                 }
-                if (call_count == 0) {
-                        INIT_LIST_HEAD (&paused_calls);
-                        afr_get_resumable_calls (this, fd_ctx, &paused_calls);
-                }
-        }
-        UNLOCK (&local_fd->lock);
-out:
-        if (call_count == 0) {
-                afr_resume_calls (this, &paused_calls);
-                //If the fop is paused then resume_calls will continue the fop
-                if (fop_paused)
-                        goto done;
-
-                if (local->fop_call_continue)
-                        local->fop_call_continue (frame, this);
-                else
-                        AFR_STACK_DESTROY (frame);
         }
+        UNLOCK (&local->fd->lock);
+
+        call_count = afr_frame_return (frame);
+        if (call_count == 0)
+                AFR_STACK_DESTROY (frame);
 
-done:
-        fd_unref (local_fd);
         return 0;
 }
 
-int
-afr_fix_open (call_frame_t *frame, xlator_t *this, afr_fd_ctx_t *fd_ctx,
-              int need_open_count, int *need_open)
+
+static int
+afr_fd_ctx_need_open (fd_t *fd, xlator_t *this, unsigned char *need_open)
 {
-        afr_local_t       *local = NULL;
-        afr_private_t     *priv  = NULL;
-        int               i      = 0;
-        call_frame_t      *open_frame = NULL;
-        afr_local_t      *open_local = NULL;
-        int               ret    = -1;
-        ia_type_t         ia_type = IA_INVAL;
-        int32_t           op_errno = 0;
-
-        GF_ASSERT (fd_ctx);
-        GF_ASSERT (need_open_count > 0);
-        GF_ASSERT (need_open);
+	afr_fd_ctx_t *fd_ctx = NULL;
+	afr_private_t *priv = NULL;
+	int i = 0;
+	int count = 0;
+
+	priv = this->private;
+
+	fd_ctx = afr_fd_ctx_get (fd, this);
+	if (!fd_ctx)
+		return 0;
+
+	LOCK (&fd->lock);
+	{
+		for (i = 0; i < priv->child_count; i++) {
+			if (fd_ctx->opened_on[i] == AFR_FD_NOT_OPENED &&
+			    priv->child_up[i]) {
+				fd_ctx->opened_on[i] = AFR_FD_OPENING;
+				need_open[i] = 1;
+				count++;
+			} else {
+				need_open[i] = 0;
+			}
+		}
+	}
+	UNLOCK (&fd->lock);
+
+	return count;
+}
+
+
+void
+afr_fix_open (fd_t *fd, xlator_t *this)
+{
+        afr_private_t *priv    = NULL;
+        int           i        = 0;
+        call_frame_t  *frame   = NULL;
+        afr_local_t   *local   = NULL;
+        int           ret      = -1;
+        int32_t       op_errno = 0;
+        afr_fd_ctx_t  *fd_ctx  = NULL;
+	unsigned char *need_open = NULL;
+	int call_count = 0;
 
-        local = frame->local;
         priv  = this->private;
-        if (!local->fop_call_continue) {
-                open_frame = copy_frame (frame);
-                if (!open_frame) {
-                        ret = -ENOMEM;
-                        goto out;
-                }
-                AFR_LOCAL_ALLOC_OR_GOTO (open_frame->local, out);
-                open_local = open_frame->local;
-                ret = afr_local_init (open_local, priv, &op_errno);
-                if (ret < 0)
-                        goto out;
-                loc_copy (&open_local->loc, &local->loc);
-                open_local->fd = fd_ref (local->fd);
-        } else {
-                ret = 0;
-                open_frame = frame;
-                open_local = local;
-        }
 
-        open_local->call_count = need_open_count;
+        if (!afr_is_fd_fixable (fd))
+                goto out;
+
+        fd_ctx = afr_fd_ctx_get (fd, this);
+        if (!fd_ctx)
+                goto out;
+
+	need_open = alloca0 (priv->child_count);
+
+	call_count = afr_fd_ctx_need_open (fd, this, need_open);
+	if (!call_count)
+		goto out;
+
+        frame = create_frame (this, this->ctx->pool);
+        if (!frame)
+                goto out;
+
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
+
+        local->loc.inode = inode_ref (fd->inode);
+        ret = loc_path (&local->loc, NULL);
+        if (ret < 0)
+                goto out;
+
+        local->fd = fd_ref (fd);
+	local->fd_ctx = fd_ctx;
+
+        local->call_count = call_count;
 
         gf_log (this->name, GF_LOG_DEBUG, "need open count: %d",
-                need_open_count);
+                call_count);
 
-        ia_type = open_local->fd->inode->ia_type;
-        GF_ASSERT (ia_type != IA_INVAL);
         for (i = 0; i < priv->child_count; i++) {
                 if (!need_open[i])
                         continue;
-                if (IA_IFDIR == ia_type) {
+
+                if (IA_IFDIR == fd->inode->ia_type) {
                         gf_log (this->name, GF_LOG_DEBUG,
                                 "opening fd for dir %s on subvolume %s",
                                 local->loc.path, priv->children[i]->name);
 
-                        STACK_WIND_COOKIE (open_frame, afr_openfd_fix_open_cbk,
+                        STACK_WIND_COOKIE (frame, afr_openfd_fix_open_cbk,
                                            (void*) (long) i,
                                            priv->children[i],
                                            priv->children[i]->fops->opendir,
-                                           &open_local->loc, open_local->fd,
+                                           &local->loc, local->fd,
                                            NULL);
                 } else {
                         gf_log (this->name, GF_LOG_DEBUG,
                                 "opening fd for file %s on subvolume %s",
                                 local->loc.path, priv->children[i]->name);
 
-                        STACK_WIND_COOKIE (open_frame, afr_openfd_fix_open_cbk,
+                        STACK_WIND_COOKIE (frame, afr_openfd_fix_open_cbk,
                                            (void *)(long) i,
                                            priv->children[i],
                                            priv->children[i]->fops->open,
-                                           &open_local->loc,
+                                           &local->loc,
                                            fd_ctx->flags & (~O_TRUNC),
-                                           open_local->fd, NULL);
+                                           local->fd, NULL);
                 }
 
+		if (!--call_count)
+			break;
         }
-        op_errno = 0;
-        ret = 0;
+
+	return;
 out:
-        if (op_errno)
-                ret = -op_errno;
-        if (ret && open_frame)
-                AFR_STACK_DESTROY (open_frame);
-        return ret;
+        if (frame)
+                AFR_STACK_DESTROY (frame);
 }
diff --git a/xlators/cluster/afr/src/afr-read-txn.c b/xlators/cluster/afr/src/afr-read-txn.c
new file mode 100644
index 000000000..186f68c33
--- /dev/null
+++ b/xlators/cluster/afr/src/afr-read-txn.c
@@ -0,0 +1,239 @@
+/*
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include "afr.h"
+#include "afr-transaction.h"
+
+int
+afr_read_txn_next_subvol (call_frame_t *frame, xlator_t *this)
+{
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	int i = 0;
+	int subvol = -1;
+
+	local = frame->local;
+	priv = this->private;
+
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!local->readable[i]) {
+			/* don't even bother trying here.
+			   just mark as attempted and move on. */
+			local->read_attempted[i] = 1;
+			continue;
+		}
+
+		if (!local->read_attempted[i]) {
+			subvol = i;
+			break;
+		}
+	}
+
+	/* If no more subvols were available for reading, we leave
+	   @subvol as -1, which is an indication we have run out of
+	   readable subvols. */
+	if (subvol != -1)
+		local->read_attempted[subvol] = 1;
+	local->readfn (frame, this, subvol);
+
+	return 0;
+}
+
+
+int
+afr_read_txn_refresh_done (call_frame_t *frame, xlator_t *this, int err)
+{
+	afr_local_t *local = NULL;
+	int read_subvol = 0;
+	int event_generation = 0;
+	inode_t *inode = NULL;
+	int ret = -1;
+
+	local = frame->local;
+	inode = local->inode;
+
+	if (err) {
+		local->op_errno = -err;
+		local->op_ret = -1;
+		read_subvol = -1;
+		goto readfn;
+	}
+
+	ret = afr_inode_read_subvol_type_get (inode, this, local->readable,
+					      &event_generation,
+					      local->transaction.type);
+
+	if (ret == -1 || !event_generation) {
+		/* Even after refresh, we don't have a good
+		   read subvolume. Time to bail */
+		local->op_ret = -1;
+		local->op_errno = EIO;
+		read_subvol = -1;
+		goto readfn;
+	}
+
+	read_subvol = afr_read_subvol_select_by_policy (inode, this,
+							local->readable);
+
+	if (read_subvol == -1) {
+		local->op_ret = -1;
+		local->op_errno = EIO;
+		goto readfn;
+	}
+
+	if (local->read_attempted[read_subvol]) {
+		afr_read_txn_next_subvol (frame, this);
+		return 0;
+	}
+
+	local->read_attempted[read_subvol] = 1;
+readfn:
+	local->readfn (frame, this, read_subvol);
+
+	return 0;
+}
+
+
+int
+afr_read_txn_continue (call_frame_t *frame, xlator_t *this, int subvol)
+{
+	afr_local_t *local = NULL;
+
+	local = frame->local;
+
+	if (!local->refreshed) {
+		local->refreshed = _gf_true;
+		afr_inode_refresh (frame, this, local->inode,
+				   afr_read_txn_refresh_done);
+	} else {
+		afr_read_txn_next_subvol (frame, this);
+	}
+
+	return 0;
+}
+
+
+/* afr_read_txn_wipe:
+
+   clean internal variables in @local in order to make
+   it possible to call afr_read_txn() multiple times from
+   the same frame
+*/
+
+void
+afr_read_txn_wipe (call_frame_t *frame, xlator_t *this)
+{
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	int i = 0;
+
+	local = frame->local;
+	priv = this->private;
+
+	local->readfn = NULL;
+
+	if (local->inode)
+		inode_unref (local->inode);
+
+	for (i = 0; i < priv->child_count; i++) {
+		local->read_attempted[i] = 0;
+		local->readable[i] = 0;
+	}
+}
+
+
+/*
+  afr_read_txn:
+
+  This is the read transaction function. The way it works:
+
+  - Determine read-subvolume from inode ctx.
+
+  - If read-subvolume's generation was stale, refresh ctx once by
+    calling afr_inode_refresh()
+
+    Else make an attempt to read on read-subvolume.
+
+  - If attempted read on read-subvolume fails, refresh ctx once
+    by calling afr_inode_refresh()
+
+  - After ctx refresh, query read-subvolume freshly and attempt
+    read once.
+
+  - If read fails, try every other readable[] subvolume before
+    finally giving up. readable[] elements are set by afr_inode_refresh()
+    based on dirty and pending flags.
+
+  - If file is in split brain in the backend, generation will be
+    kept 0 by afr_inode_refresh() and readable[] will be set 0 for
+    all elements. Therefore reads always fail.
+*/
+
+int
+afr_read_txn (call_frame_t *frame, xlator_t *this, inode_t *inode,
+	      afr_read_txn_wind_t readfn, afr_transaction_type type)
+{
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	int read_subvol = -1;
+	int event_generation = 0;
+	int ret = -1;
+
+	priv = this->private;
+	local = frame->local;
+
+	afr_read_txn_wipe (frame, this);
+
+	local->readfn = readfn;
+	local->inode = inode_ref (inode);
+
+	local->transaction.type = type;
+	ret = afr_inode_read_subvol_type_get (inode, this, local->readable,
+					      &event_generation, type);
+	if (ret == -1)
+		/* very first transaction on this inode */
+		goto refresh;
+
+	if (local->event_generation != event_generation)
+		/* servers have disconnected / reconnected, and possibly
+		   rebooted, very likely changing the state of freshness
+		   of copies */
+		goto refresh;
+
+	read_subvol = afr_read_subvol_select_by_policy (inode, this,
+							local->readable);
+
+	if (read_subvol < 0 || read_subvol > priv->child_count) {
+		gf_log (this->name, GF_LOG_WARNING, "Unreadable subvolume %d "
+			"found with event generation %d", read_subvol,
+			event_generation);
+		goto refresh;
+	}
+
+	if (!local->child_up[read_subvol]) {
+		/* should never happen, just in case */
+		gf_log (this->name, GF_LOG_WARNING, "subvolume %d is the "
+			"read subvolume in this generation, but is not up",
+			read_subvol);
+		goto refresh;
+	}
+
+	local->read_attempted[read_subvol] = 1;
+
+	local->readfn (frame, this, read_subvol);
+
+	return 0;
+
+refresh:
+	afr_inode_refresh (frame, this, inode, afr_read_txn_refresh_done);
+
+	return 0;
+}
diff --git a/xlators/cluster/afr/src/afr-self-heal-algorithm.c b/xlators/cluster/afr/src/afr-self-heal-algorithm.c
deleted file mode 100644
index 30e117b65..000000000
--- a/xlators/cluster/afr/src/afr-self-heal-algorithm.c
+++ /dev/null
@@ -1,810 +0,0 @@
-/*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-
-#include <openssl/md5.h>
-#include "glusterfs.h"
-#include "afr.h"
-#include "xlator.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "stack.h"
-#include "list.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include "compat-errno.h"
-#include "compat.h"
-#include "byte-order.h"
-
-#include "afr-transaction.h"
-#include "afr-self-heal.h"
-#include "afr-self-heal-common.h"
-#include "afr-self-heal-algorithm.h"
-
-/*
-  This file contains the various self-heal algorithms
-*/
-
-static int
-sh_loop_driver (call_frame_t *sh_frame, xlator_t *this,
-                gf_boolean_t is_first_call, call_frame_t *old_loop_frame);
-static int
-sh_loop_return (call_frame_t *sh_frame, xlator_t *this, call_frame_t *loop_frame,
-                int32_t op_ret, int32_t op_errno);
-static int
-sh_destroy_frame (call_frame_t *frame, xlator_t *this)
-{
-        if (!frame)
-                goto out;
-
-        AFR_STACK_DESTROY (frame);
-out:
-        return 0;
-}
-
-static void
-sh_private_cleanup (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t             *local   = NULL;
-        afr_self_heal_t         *sh      = NULL;
-        afr_sh_algo_private_t   *sh_priv = NULL;
-
-        local = frame->local;
-        sh    = &local->self_heal;
-
-        sh_priv = sh->private;
-        if (sh_priv)
-                GF_FREE (sh_priv);
-}
-
-static int
-sh_number_of_writes_needed (unsigned char *write_needed, int child_count)
-{
-        int writes = 0;
-        int i      = 0;
-
-        for (i = 0; i < child_count; i++) {
-                if (write_needed[i])
-                        writes++;
-        }
-
-        return writes;
-}
-
-
-static int
-sh_loop_driver_done (call_frame_t *sh_frame, xlator_t *this,
-                     call_frame_t *last_loop_frame)
-{
-        afr_local_t             *local        = NULL;
-        afr_self_heal_t         *sh           = NULL;
-        afr_sh_algo_private_t   *sh_priv      = NULL;
-        int32_t                 total_blocks = 0;
-        int32_t                 diff_blocks  = 0;
-
-        local        = sh_frame->local;
-        sh           = &local->self_heal;
-        sh_priv      = sh->private;
-        if (sh_priv) {
-                total_blocks = sh_priv->total_blocks;
-                diff_blocks  = sh_priv->diff_blocks;
-        }
-
-        sh_private_cleanup (sh_frame, this);
-        if (sh->op_failed) {
-                GF_ASSERT (!last_loop_frame);
-                //loop_finish should have happened and the old_loop should be NULL
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "self-heal aborting on %s",
-                        local->loc.path);
-
-                local->self_heal.algo_abort_cbk (sh_frame, this);
-        } else {
-                GF_ASSERT (last_loop_frame);
-                if (diff_blocks == total_blocks) {
-                        gf_log (this->name, GF_LOG_DEBUG, "full self-heal "
-                                "completed on %s",local->loc.path);
-                } else {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "diff self-heal on %s: completed. "
-                                "(%d blocks of %d were different (%.2f%%))",
-                                local->loc.path, diff_blocks, total_blocks,
-                                ((diff_blocks * 1.0)/total_blocks) * 100);
-                }
-
-                sh->old_loop_frame = last_loop_frame;
-                local->self_heal.algo_completion_cbk (sh_frame, this);
-        }
-
-        return 0;
-}
-
-int
-sh_loop_finish (call_frame_t *loop_frame, xlator_t *this)
-{
-        afr_local_t             *loop_local = NULL;
-        afr_self_heal_t         *loop_sh = NULL;
-
-        if (!loop_frame)
-                goto out;
-
-        loop_local = loop_frame->local;
-        if (loop_local) {
-                loop_sh = &loop_local->self_heal;
-        }
-
-        if (loop_sh && loop_sh->data_lock_held) {
-                afr_sh_data_unlock (loop_frame, this,
-                                    sh_destroy_frame);
-        } else {
-                sh_destroy_frame (loop_frame, this);
-        }
-out:
-        return 0;
-}
-
-static int
-sh_loop_lock_success (call_frame_t *loop_frame, xlator_t *this)
-{
-        afr_local_t                 *loop_local = NULL;
-        afr_self_heal_t             *loop_sh    = NULL;
-
-        loop_local = loop_frame->local;
-        loop_sh = &loop_local->self_heal;
-
-        sh_loop_finish (loop_sh->old_loop_frame, this);
-        loop_sh->old_loop_frame = NULL;
-
-        gf_log (this->name, GF_LOG_DEBUG, "Acquired lock for range %"PRIu64
-                " %"PRIu64, loop_sh->offset, loop_sh->block_size);
-        loop_sh->data_lock_held = _gf_true;
-        loop_sh->sh_data_algo_start (loop_frame, this);
-        return 0;
-}
-
-static int
-sh_loop_lock_failure (call_frame_t *loop_frame, xlator_t *this)
-{
-        call_frame_t                *sh_frame = NULL;
-        afr_local_t                 *loop_local = NULL;
-        afr_self_heal_t             *loop_sh    = NULL;
-
-        loop_local = loop_frame->local;
-        loop_sh = &loop_local->self_heal;
-        sh_frame = loop_sh->sh_frame;
-
-        gf_log (this->name, GF_LOG_ERROR, "failed lock for range %"PRIu64
-                " %"PRIu64, loop_sh->offset, loop_sh->block_size);
-        sh_loop_finish (loop_sh->old_loop_frame, this);
-        loop_sh->old_loop_frame = NULL;
-        sh_loop_return (sh_frame, this, loop_frame, -1, ENOTCONN);
-        return 0;
-}
-
-static int
-sh_loop_frame_create (call_frame_t *sh_frame, xlator_t *this,
-                      call_frame_t *old_loop_frame, call_frame_t **loop_frame)
-{
-        call_frame_t                *new_loop_frame = NULL;
-        afr_local_t                 *local          = NULL;
-        afr_self_heal_t             *sh             = NULL;
-        afr_local_t                 *new_loop_local = NULL;
-        afr_self_heal_t             *new_loop_sh    = NULL;
-        afr_private_t               *priv           = NULL;
-
-        GF_ASSERT (sh_frame);
-        GF_ASSERT (loop_frame);
-
-        *loop_frame = NULL;
-        local   = sh_frame->local;
-        sh      = &local->self_heal;
-        priv    = this->private;
-
-        new_loop_frame = copy_frame (sh_frame);
-        if (!new_loop_frame)
-                goto out;
-        //We want the frame to have same lk_owner as sh_frame
-        //so that locks translator allows conflicting locks
-        new_loop_local = afr_local_copy (local, this);
-        if (!new_loop_local)
-                goto out;
-        new_loop_frame->local = new_loop_local;
-
-        new_loop_sh = &new_loop_local->self_heal;
-        new_loop_sh->sources = memdup (sh->sources,
-                                       priv->child_count * sizeof (*sh->sources));
-        if (!new_loop_sh->sources)
-                goto out;
-        new_loop_sh->write_needed = GF_CALLOC (priv->child_count,
-                                               sizeof (*new_loop_sh->write_needed),
-                                               gf_afr_mt_char);
-        if (!new_loop_sh->write_needed)
-                goto out;
-        new_loop_sh->checksum = GF_CALLOC (priv->child_count, MD5_DIGEST_LENGTH,
-                                           gf_afr_mt_uint8_t);
-        if (!new_loop_sh->checksum)
-                goto out;
-        new_loop_sh->inode      = inode_ref (sh->inode);
-        new_loop_sh->sh_data_algo_start = sh->sh_data_algo_start;
-        new_loop_sh->source = sh->source;
-        new_loop_sh->active_sinks = sh->active_sinks;
-        new_loop_sh->healing_fd = fd_ref (sh->healing_fd);
-        new_loop_sh->file_has_holes = sh->file_has_holes;
-        new_loop_sh->old_loop_frame = old_loop_frame;
-        new_loop_sh->sh_frame = sh_frame;
-        *loop_frame = new_loop_frame;
-        return 0;
-out:
-        sh_destroy_frame (new_loop_frame, this);
-        return -ENOMEM;
-}
-
-static int
-sh_loop_start (call_frame_t *sh_frame, xlator_t *this, off_t offset,
-               call_frame_t *old_loop_frame)
-{
-        call_frame_t                *new_loop_frame = NULL;
-        afr_local_t                 *local          = NULL;
-        afr_self_heal_t             *sh             = NULL;
-        afr_local_t                 *new_loop_local = NULL;
-        afr_self_heal_t             *new_loop_sh    = NULL;
-        int                         ret             = 0;
-
-        GF_ASSERT (sh_frame);
-
-        local   = sh_frame->local;
-        sh      = &local->self_heal;
-
-        ret = sh_loop_frame_create (sh_frame, this, old_loop_frame,
-                                    &new_loop_frame);
-        if (ret)
-                goto out;
-        new_loop_local = new_loop_frame->local;
-        new_loop_sh = &new_loop_local->self_heal;
-        new_loop_sh->offset = offset;
-        new_loop_sh->block_size = sh->block_size;
-        afr_sh_data_lock (new_loop_frame, this, offset, new_loop_sh->block_size,
-                          sh_loop_lock_success, sh_loop_lock_failure);
-        return 0;
-out:
-        sh->op_failed = 1;
-        if (old_loop_frame)
-                sh_loop_finish (old_loop_frame, this);
-        sh_loop_return (sh_frame, this, new_loop_frame, -1, ENOMEM);
-        return 0;
-}
-
-static int
-sh_loop_driver (call_frame_t *sh_frame, xlator_t *this,
-                gf_boolean_t is_first_call, call_frame_t *old_loop_frame)
-{
-        afr_local_t *               local          = NULL;
-        afr_self_heal_t *           sh             = NULL;
-        afr_sh_algo_private_t       *sh_priv        = NULL;
-        gf_boolean_t                is_driver_done = _gf_false;
-        blksize_t                   block_size     = 0;
-        int                         loop           = 0;
-        off_t                       offset         = 0;
-        afr_private_t               *priv          = NULL;
-
-        priv    = this->private;
-        local   = sh_frame->local;
-        sh      = &local->self_heal;
-        sh_priv = sh->private;
-
-        LOCK (&sh_priv->lock);
-        {
-                if (!is_first_call)
-                        sh_priv->loops_running--;
-                offset = sh_priv->offset;
-                block_size = sh->block_size;
-                while ((!sh->eof_reached) && (0 == sh->op_failed) &&
-                       (sh_priv->loops_running < priv->data_self_heal_window_size)
-                       && (sh_priv->offset < sh->file_size)) {
-
-                        loop++;
-                        sh_priv->offset += block_size;
-                        sh_priv->loops_running++;
-
-                        if (!is_first_call)
-                                break;
-                }
-                if (0 == sh_priv->loops_running) {
-                        is_driver_done = _gf_true;
-                }
-        }
-        UNLOCK (&sh_priv->lock);
-
-        if (0 == loop) {
-                //loop finish does unlock, but the erasing of the pending
-                //xattrs needs to happen before that so do not finish the loop
-                if (is_driver_done && !sh->op_failed)
-                        goto driver_done;
-                if (old_loop_frame) {
-                        sh_loop_finish (old_loop_frame, this);
-                        old_loop_frame = NULL;
-                }
-        }
-
-        //If we have more loops to form we should finish previous loop after
-        //the next loop lock
-        while (loop--) {
-                if (sh->op_failed) {
-                        // op failed in other loop, stop spawning more loops
-                        if (old_loop_frame) {
-                                sh_loop_finish (old_loop_frame, this);
-                                old_loop_frame = NULL;
-                        }
-                        sh_loop_driver (sh_frame, this, _gf_false, NULL);
-                } else {
-                        gf_log (this->name, GF_LOG_TRACE, "spawning a loop "
-                                "for offset %"PRId64, offset);
-
-                        sh_loop_start (sh_frame, this, offset, old_loop_frame);
-                        old_loop_frame = NULL;
-                        offset += block_size;
-                }
-        }
-
-driver_done:
-        if (is_driver_done) {
-                sh_loop_driver_done (sh_frame, this, old_loop_frame);
-        }
-        return 0;
-}
-
-static int
-sh_loop_return (call_frame_t *sh_frame, xlator_t *this, call_frame_t *loop_frame,
-                int32_t op_ret, int32_t op_errno)
-{
-        afr_local_t *               loop_local = NULL;
-        afr_self_heal_t *           loop_sh    = NULL;
-        afr_local_t *               sh_local = NULL;
-        afr_self_heal_t            *sh       = NULL;
-
-        sh_local = sh_frame->local;
-        sh       = &sh_local->self_heal;
-
-        if (loop_frame) {
-                loop_local = loop_frame->local;
-                if (loop_local)
-                        loop_sh    = &loop_local->self_heal;
-                if (loop_sh)
-                        gf_log (this->name, GF_LOG_TRACE, "loop for offset "
-                                "%"PRId64" returned", loop_sh->offset);
-        }
-
-        if (op_ret == -1) {
-                sh->op_failed = 1;
-                afr_sh_set_error (sh, op_errno);
-                if (loop_frame) {
-                        sh_loop_finish (loop_frame, this);
-                        loop_frame = NULL;
-                }
-        }
-
-        sh_loop_driver (sh_frame, this, _gf_false, loop_frame);
-
-        return 0;
-}
-
-static int
-sh_loop_write_cbk (call_frame_t *loop_frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                   struct iatt *postbuf, dict_t *xdata)
-{
-        afr_private_t *             priv        = NULL;
-        afr_local_t *               loop_local    = NULL;
-        afr_self_heal_t *           loop_sh       = NULL;
-        call_frame_t               *sh_frame    = NULL;
-        afr_local_t *               sh_local    = NULL;
-        afr_self_heal_t            *sh          = NULL;
-        int                         call_count  = 0;
-        int                         child_index = 0;
-
-        priv     = this->private;
-        loop_local = loop_frame->local;
-        loop_sh    = &loop_local->self_heal;
-
-        sh_frame = loop_sh->sh_frame;
-        sh_local = sh_frame->local;
-        sh       = &sh_local->self_heal;
-
-        child_index =  (long) cookie;
-
-        gf_log (this->name, GF_LOG_TRACE,
-                "wrote %d bytes of data from %s to child %d, offset %"PRId64"",
-                op_ret, sh_local->loc.path, child_index, loop_sh->offset);
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "write to %s failed on subvolume %s (%s)",
-                        sh_local->loc.path,
-                        priv->children[child_index]->name,
-                        strerror (op_errno));
-
-                sh->op_failed = 1;
-                afr_sh_set_error (loop_sh, op_errno);
-        }
-
-        call_count = afr_frame_return (loop_frame);
-
-        if (call_count == 0) {
-                sh_loop_return (sh_frame, this, loop_frame,
-                                loop_sh->op_ret, loop_sh->op_errno);
-        }
-
-        return 0;
-}
-
-static void
-sh_prune_writes_needed (call_frame_t *sh_frame, call_frame_t *loop_frame,
-                        afr_private_t *priv)
-{
-        afr_local_t     *sh_local     = NULL;
-        afr_self_heal_t *sh           = NULL;
-        afr_local_t     *loop_local   = NULL;
-        afr_self_heal_t *loop_sh      = NULL;
-        int             i             = 0;
-
-        sh_local   = sh_frame->local;
-        sh         = &sh_local->self_heal;
-
-        if (!strcmp (sh->algo->name, "diff"))
-                return;
-
-        loop_local = loop_frame->local;
-        loop_sh    = &loop_local->self_heal;
-
-        /* full self-heal guarantees there exists atleast 1 file with size 0
-         * That means for other files we can preserve holes that come after
-         * its size before 'trim'
-         */
-        for (i = 0; i < priv->child_count; i++) {
-                if (loop_sh->write_needed[i] &&
-                    ((loop_sh->offset + 1) > sh->buf[i].ia_size))
-                        loop_sh->write_needed[i] = 0;
-        }
-}
-
-static int
-sh_loop_read_cbk (call_frame_t *loop_frame, void *cookie,
-                  xlator_t *this, int32_t op_ret, int32_t op_errno,
-                  struct iovec *vector, int32_t count, struct iatt *buf,
-                  struct iobref *iobref, dict_t *xdata)
-{
-        afr_private_t *               priv       = NULL;
-        afr_local_t *                 loop_local   = NULL;
-        afr_self_heal_t *             loop_sh      = NULL;
-        call_frame_t                 *sh_frame   = NULL;
-        int                           i          = 0;
-        int                           call_count = 0;
-        afr_local_t *                 sh_local   = NULL;
-        afr_self_heal_t *             sh      = NULL;
-
-        priv       = this->private;
-        loop_local = loop_frame->local;
-        loop_sh    = &loop_local->self_heal;
-
-        sh_frame = loop_sh->sh_frame;
-        sh_local = sh_frame->local;
-        sh       = &sh_local->self_heal;
-
-        gf_log (this->name, GF_LOG_TRACE,
-                "read %d bytes of data from %s, offset %"PRId64"",
-                op_ret, loop_local->loc.path, loop_sh->offset);
-
-        if (op_ret <= 0) {
-                if (op_ret < 0) {
-                        sh->op_failed = 1;
-                        gf_log (this->name, GF_LOG_ERROR, "read failed on %d "
-                                "for %s reason :%s", sh->source,
-                                sh_local->loc.path, strerror (errno));
-                } else {
-                        sh->eof_reached = _gf_true;
-                        gf_log (this->name, GF_LOG_DEBUG, "Eof reached for %s",
-                                sh_local->loc.path);
-                }
-                sh_loop_return (sh_frame, this, loop_frame, op_ret, op_errno);
-                goto out;
-        }
-
-        if (loop_sh->file_has_holes && iov_0filled (vector, count) == 0)
-                sh_prune_writes_needed (sh_frame, loop_frame, priv);
-
-        call_count = sh_number_of_writes_needed (loop_sh->write_needed,
-                                                 priv->child_count);
-        if (call_count == 0) {
-                sh_loop_return (sh_frame, this, loop_frame, 0, 0);
-                goto out;
-        }
-        loop_local->call_count = call_count;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (!loop_sh->write_needed[i])
-                        continue;
-                STACK_WIND_COOKIE (loop_frame, sh_loop_write_cbk,
-                                   (void *) (long) i,
-                                   priv->children[i],
-                                   priv->children[i]->fops->writev,
-                                   loop_sh->healing_fd, vector, count,
-                                   loop_sh->offset, 0, iobref, NULL);
-
-                if (!--call_count)
-                        break;
-        }
-
-out:
-        return 0;
-}
-
-
-static int
-sh_loop_read (call_frame_t *loop_frame, xlator_t *this)
-{
-        afr_private_t           *priv       = NULL;
-        afr_local_t             *loop_local   = NULL;
-        afr_self_heal_t         *loop_sh      = NULL;
-
-        priv     = this->private;
-        loop_local = loop_frame->local;
-        loop_sh    = &loop_local->self_heal;
-
-        STACK_WIND_COOKIE (loop_frame, sh_loop_read_cbk,
-                           (void *) (long) loop_sh->source,
-                           priv->children[loop_sh->source],
-                           priv->children[loop_sh->source]->fops->readv,
-                           loop_sh->healing_fd, loop_sh->block_size,
-                           loop_sh->offset, 0, NULL);
-
-        return 0;
-}
-
-
-static int
-sh_diff_checksum_cbk (call_frame_t *loop_frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno,
-                      uint32_t weak_checksum, uint8_t *strong_checksum,
-                      dict_t *xdata)
-{
-        afr_private_t                 *priv         = NULL;
-        afr_local_t                   *loop_local   = NULL;
-        afr_self_heal_t               *loop_sh      = NULL;
-        call_frame_t                  *sh_frame     = NULL;
-        afr_local_t                   *sh_local     = NULL;
-        afr_self_heal_t               *sh           = NULL;
-        afr_sh_algo_private_t         *sh_priv      = NULL;
-        int                           child_index  = 0;
-        int                           call_count   = 0;
-        int                           i            = 0;
-        int                           write_needed = 0;
-
-        priv  = this->private;
-
-        loop_local = loop_frame->local;
-        loop_sh    = &loop_local->self_heal;
-
-        sh_frame = loop_sh->sh_frame;
-        sh_local = sh_frame->local;
-        sh       = &sh_local->self_heal;
-
-        sh_priv = sh->private;
-
-        child_index = (long) cookie;
-
-        if (op_ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "checksum on %s failed on subvolume %s (%s)",
-                        sh_local->loc.path, priv->children[child_index]->name,
-                        strerror (op_errno));
-                sh->op_failed = 1;
-        } else {
-                memcpy (loop_sh->checksum + child_index * MD5_DIGEST_LENGTH,
-                        strong_checksum, MD5_DIGEST_LENGTH);
-        }
-
-        call_count = afr_frame_return (loop_frame);
-
-        if (call_count == 0) {
-                for (i = 0; i < priv->child_count; i++) {
-                        if (sh->sources[i] || !sh_local->child_up[i])
-                                continue;
-
-                        if (memcmp (loop_sh->checksum + (i * MD5_DIGEST_LENGTH),
-                                    loop_sh->checksum + (sh->source * MD5_DIGEST_LENGTH),
-                                    MD5_DIGEST_LENGTH)) {
-                                /*
-                                  Checksums differ, so this block
-                                  must be written to this sink
-                                */
-
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                        "checksum on subvolume %s at offset %"
-                                        PRId64" differs from that on source",
-                                        priv->children[i]->name, loop_sh->offset);
-
-                                write_needed = loop_sh->write_needed[i] = 1;
-                        }
-                }
-
-                LOCK (&sh_priv->lock);
-                {
-                        sh_priv->total_blocks++;
-                        if (write_needed)
-                                sh_priv->diff_blocks++;
-                }
-                UNLOCK (&sh_priv->lock);
-
-                if (write_needed && !sh->op_failed) {
-                        sh_loop_read (loop_frame, this);
-                } else {
-                        sh_loop_return (sh_frame, this, loop_frame,
-                                        op_ret, op_errno);
-                }
-        }
-
-        return 0;
-}
-
-static int
-sh_diff_checksum (call_frame_t *loop_frame, xlator_t *this)
-{
-        afr_private_t           *priv         = NULL;
-        afr_local_t             *loop_local   = NULL;
-        afr_self_heal_t         *loop_sh      = NULL;
-        int                     call_count    = 0;
-        int                     i             = 0;
-
-        priv         = this->private;
-        loop_local   = loop_frame->local;
-        loop_sh      = &loop_local->self_heal;
-
-        call_count = loop_sh->active_sinks + 1;  /* sinks and source */
-
-        loop_local->call_count = call_count;
-
-        STACK_WIND_COOKIE (loop_frame, sh_diff_checksum_cbk,
-                           (void *) (long) loop_sh->source,
-                           priv->children[loop_sh->source],
-                           priv->children[loop_sh->source]->fops->rchecksum,
-                           loop_sh->healing_fd,
-                           loop_sh->offset, loop_sh->block_size, NULL);
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (loop_sh->sources[i] || !loop_local->child_up[i])
-                        continue;
-
-                STACK_WIND_COOKIE (loop_frame, sh_diff_checksum_cbk,
-                                   (void *) (long) i,
-                                   priv->children[i],
-                                   priv->children[i]->fops->rchecksum,
-                                   loop_sh->healing_fd,
-                                   loop_sh->offset, loop_sh->block_size, NULL);
-
-                if (!--call_count)
-                        break;
-        }
-
-        return 0;
-}
-
-static int
-sh_full_read_write_to_sinks (call_frame_t *loop_frame, xlator_t *this)
-{
-        afr_private_t           *priv         = NULL;
-        afr_local_t             *loop_local   = NULL;
-        afr_self_heal_t         *loop_sh      = NULL;
-        int                     i             = 0;
-
-        priv         = this->private;
-        loop_local   = loop_frame->local;
-        loop_sh      = &loop_local->self_heal;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (loop_sh->sources[i] || !loop_local->child_up[i])
-                        continue;
-                loop_sh->write_needed[i] = 1;
-        }
-        sh_loop_read (loop_frame, this);
-        return 0;
-}
-
-afr_sh_algo_private_t*
-afr_sh_priv_init ()
-{
-        afr_sh_algo_private_t   *sh_priv = NULL;
-
-        sh_priv = GF_CALLOC (1, sizeof (*sh_priv),
-                             gf_afr_mt_afr_private_t);
-        if (!sh_priv)
-                goto out;
-
-        LOCK_INIT (&sh_priv->lock);
-out:
-        return sh_priv;
-}
-
-void
-afr_sh_transfer_lock (call_frame_t *dst, call_frame_t *src,
-                      unsigned int child_count)
-{
-        afr_local_t             *dst_local   = NULL;
-        afr_self_heal_t         *dst_sh      = NULL;
-        afr_local_t             *src_local   = NULL;
-        afr_self_heal_t         *src_sh      = NULL;
-
-        dst_local = dst->local;
-        dst_sh = &dst_local->self_heal;
-        src_local = src->local;
-        src_sh = &src_local->self_heal;
-        GF_ASSERT (src_sh->data_lock_held);
-        GF_ASSERT (!dst_sh->data_lock_held);
-        afr_lk_transfer_datalock (dst, src, child_count);
-        src_sh->data_lock_held = _gf_false;
-        dst_sh->data_lock_held = _gf_true;
-}
-
-int
-afr_sh_start_loops (call_frame_t *sh_frame, xlator_t *this,
-                    afr_sh_algo_fn sh_data_algo_start)
-{
-        call_frame_t            *first_loop_frame = NULL;
-        afr_local_t             *local   = NULL;
-        afr_self_heal_t         *sh      = NULL;
-        int                     ret      = 0;
-        afr_private_t           *priv    = NULL;
-
-        local = sh_frame->local;
-        sh    = &local->self_heal;
-        priv  = this->private;
-
-        sh->sh_data_algo_start = sh_data_algo_start;
-        local->call_count = 0;
-        ret = sh_loop_frame_create (sh_frame, this, NULL, &first_loop_frame);
-        if (ret)
-                goto out;
-        afr_sh_transfer_lock (first_loop_frame, sh_frame, priv->child_count);
-        sh->private = afr_sh_priv_init ();
-        if (!sh->private) {
-                ret = -1;
-                goto out;
-        }
-        sh_loop_driver (sh_frame, this, _gf_true, first_loop_frame);
-        ret = 0;
-out:
-        if (ret) {
-                sh->op_failed = 1;
-                sh_loop_driver_done (sh_frame, this, NULL);
-        }
-        return 0;
-}
-
-int
-afr_sh_algo_diff (call_frame_t *sh_frame, xlator_t *this)
-{
-        afr_sh_start_loops (sh_frame, this, sh_diff_checksum);
-        return 0;
-}
-
-int
-afr_sh_algo_full (call_frame_t *sh_frame, xlator_t *this)
-{
-        afr_sh_start_loops (sh_frame, this, sh_full_read_write_to_sinks);
-        return 0;
-}
-
-struct afr_sh_algorithm afr_self_heal_algorithms[] = {
-        {.name = "full",  .fn = afr_sh_algo_full},
-        {.name = "diff",  .fn = afr_sh_algo_diff},
-        {0, 0},
-};
diff --git a/xlators/cluster/afr/src/afr-self-heal-algorithm.h b/xlators/cluster/afr/src/afr-self-heal-algorithm.h
deleted file mode 100644
index 6b20789b1..000000000
--- a/xlators/cluster/afr/src/afr-self-heal-algorithm.h
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#ifndef __AFR_SELF_HEAL_ALGORITHM_H__
-#define __AFR_SELF_HEAL_ALGORITHM_H__
-
-typedef int (*afr_sh_algo_fn) (call_frame_t *frame,
-                               xlator_t *this);
-
-struct afr_sh_algorithm {
-        const char *name;
-        afr_sh_algo_fn fn;
-};
-
-extern struct afr_sh_algorithm afr_self_heal_algorithms[3];
-typedef struct {
-        gf_lock_t lock;
-        unsigned int loops_running;
-        off_t offset;
-
-        int32_t total_blocks;
-        int32_t diff_blocks;
-} afr_sh_algo_private_t;
-
-#endif /* __AFR_SELF_HEAL_ALGORITHM_H__ */
diff --git a/xlators/cluster/afr/src/afr-self-heal-common.c b/xlators/cluster/afr/src/afr-self-heal-common.c
index e86f82f40..4dac83113 100644
--- a/xlators/cluster/afr/src/afr-self-heal-common.c
+++ b/xlators/cluster/afr/src/afr-self-heal-common.c
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -8,2408 +8,1002 @@
   cases as published by the Free Software Foundation.
 */
 
-#include "glusterfs.h"
-#include "xlator.h"
-#include "byte-order.h"
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
 
 #include "afr.h"
-#include "afr-transaction.h"
-#include "afr-self-heal-common.h"
 #include "afr-self-heal.h"
-#include "pump.h"
+#include "byte-order.h"
 
-void
-afr_sh_reset (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        memset (sh->child_errno, 0,
-                sizeof (*sh->child_errno) * priv->child_count);
-        memset (sh->buf, 0, sizeof (*sh->buf) * priv->child_count);
-        memset (sh->parentbufs, 0,
-                sizeof (*sh->parentbufs) * priv->child_count);
-        memset (sh->success, 0, sizeof (*sh->success) * priv->child_count);
-        memset (sh->locked_nodes, 0,
-                sizeof (*sh->locked_nodes) * priv->child_count);
-        sh->active_sinks = 0;
-
-        afr_reset_xattr (sh->xattr, priv->child_count);
-}
 
-//Intersection[child]=1 if child is part of intersection
-void
-afr_children_intersection_get (int32_t *set1, int32_t *set2,
-                               int *intersection, unsigned int child_count)
+int
+afr_selfheal_post_op_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+			  int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
 {
-        int                      i = 0;
-
-        memset (intersection, 0, sizeof (*intersection) * child_count);
-        for (i = 0; i < child_count; i++) {
-                intersection[i] = afr_is_child_present (set1, child_count, i)
-                                     && afr_is_child_present (set2, child_count,
-                                                              i);
-        }
-}
+	afr_local_t *local = NULL;
 
-/**
- * select_source - select a source and return it
- */
+	local = frame->local;
 
-int
-afr_sh_select_source (int sources[], int child_count)
-{
-        int i = 0;
-        for (i = 0; i < child_count; i++)
-                if (sources[i])
-                        return i;
+	syncbarrier_wake (&local->barrier);
 
-        return -1;
+	return 0;
 }
 
-void
-afr_sh_mark_source_sinks (call_frame_t *frame, xlator_t *this)
-{
-        int              i = 0;
-        afr_local_t     *local      = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        int              active_sinks = 0;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (sh->sources[i] == 0 && local->child_up[i] == 1) {
-                        active_sinks++;
-                        sh->success[i] = 1;
-                } else if (sh->sources[i] == 1 && local->child_up[i] == 1) {
-                        sh->success[i] = 1;
-                }
-        }
-        sh->active_sinks = active_sinks;
-}
 
 int
-afr_sh_source_count (int sources[], int child_count)
+afr_selfheal_post_op (call_frame_t *frame, xlator_t *this, inode_t *inode,
+		      int subvol, dict_t *xattr)
 {
-        int i = 0;
-        int nsource = 0;
+	afr_private_t *priv = NULL;
+	afr_local_t *local = NULL;
+	loc_t loc = {0, };
 
-        for (i = 0; i < child_count; i++)
-                if (sources[i])
-                        nsource++;
-        return nsource;
-}
+	priv = this->private;
+	local = frame->local;
 
-void
-afr_sh_set_error (afr_self_heal_t *sh, int32_t op_errno)
-{
-        sh->op_ret = -1;
-        if (afr_error_more_important (sh->op_errno, op_errno))
-                sh->op_errno = op_errno;
-}
+	loc.inode = inode_ref (inode);
+	uuid_copy (loc.gfid, inode->gfid);
 
-void
-afr_sh_print_pending_matrix (int32_t *pending_matrix[], xlator_t *this)
-{
-        afr_private_t *  priv = this->private;
-        char            *buf  = NULL;
-        char            *ptr  = NULL;
-        int              i    = 0;
-        int              j    = 0;
-
-        /* 10 digits per entry + 1 space + '[' and ']' */
-        buf = GF_MALLOC (priv->child_count * 11 + 8, gf_afr_mt_char);
-
-        for (i = 0; i < priv->child_count; i++) {
-                ptr = buf;
-                ptr += sprintf (ptr, "[ ");
-                for (j = 0; j < priv->child_count; j++) {
-                        ptr += sprintf (ptr, "%d ", pending_matrix[i][j]);
-                }
-                sprintf (ptr, "]");
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "pending_matrix: %s", buf);
-        }
-
-        GF_FREE (buf);
-}
+	STACK_WIND (frame, afr_selfheal_post_op_cbk, priv->children[subvol],
+		    priv->children[subvol]->fops->xattrop, &loc,
+		    GF_XATTROP_ADD_ARRAY, xattr, NULL);
 
-void
-afr_init_pending_matrix (int32_t **pending_matrix, size_t child_count)
-{
-        int             i   = 0;
-        int             j   = 0;
-
-        GF_ASSERT (pending_matrix);
+	syncbarrier_wait (&local->barrier, 1);
 
-        for (i = 0; i < child_count; i++) {
-                for (j = 0; j < child_count; j++) {
-                        pending_matrix[i][j] = 0;
-                }
-        }
+	return 0;
 }
 
-void
-afr_mark_ignorant_subvols_as_pending (int32_t **pending_matrix,
-                                      unsigned char *ignorant_subvols,
-                                      size_t  child_count)
-{
-        int            i                = 0;
-        int            j                = 0;
-
-        GF_ASSERT (pending_matrix);
-        GF_ASSERT (ignorant_subvols);
-
-        for (i = 0; i < child_count; i++) {
-                if (ignorant_subvols[i]) {
-                        for (j = 0; j < child_count; j++) {
-                                if (!ignorant_subvols[j])
-                                        pending_matrix[j][i] += 1;
-                        }
-                }
-        }
-}
 
-int
-afr_build_pending_matrix (char **pending_key, int32_t **pending_matrix,
-                          unsigned char *ignorant_subvols,
-                          dict_t *xattr[], afr_transaction_type type,
-                          size_t child_count)
+dict_t *
+afr_selfheal_output_xattr (xlator_t *this, afr_transaction_type type,
+			   int *output_dirty, int **output_matrix, int subvol)
 {
-        /* Indexable by result of afr_index_for_transaction_type(): 0 -- 2. */
-        int32_t        pending[3]       = {0,};
-        void          *pending_raw      = NULL;
-        int            ret              = -1;
-        int            i                = 0;
-        int            j                = 0;
-        int            k                = 0;
-
-        afr_init_pending_matrix (pending_matrix, child_count);
-
-        for (i = 0; i < child_count; i++) {
-                pending_raw = NULL;
-
-                for (j = 0; j < child_count; j++) {
-                        ret = dict_get_ptr (xattr[i], pending_key[j],
-                                            &pending_raw);
-
-                        if (ret != 0) {
-                                /*
-                                 * There is no xattr present. This means this
-                                 * subvolume should be considered an 'ignorant'
-                                 * subvolume.
-                                 */
-
-                                if (ignorant_subvols)
-                                        ignorant_subvols[i] = 1;
-                                continue;
-                        }
-
-                        memcpy (pending, pending_raw, sizeof(pending));
-                        k = afr_index_for_transaction_type (type);
-
-                        pending_matrix[i][j] = ntoh32 (pending[k]);
-                }
-        }
-
-        return ret;
-}
+	dict_t *xattr = NULL;
+	afr_private_t *priv = NULL;
+	int j = 0;
+	int idx = 0;
+	int ret = 0;
+	int *raw = 0;
 
-typedef enum {
-        AFR_NODE_INVALID,
-        AFR_NODE_INNOCENT,
-        AFR_NODE_FOOL,
-        AFR_NODE_WISE,
-} afr_node_type;
+	priv = this->private;
+	idx = afr_index_for_transaction_type (type);
 
-typedef struct {
-        afr_node_type type;
-        int           wisdom;
-} afr_node_character;
+	xattr = dict_new ();
+	if (!xattr)
+		return NULL;
 
+	if (output_dirty[subvol]) {
+		/* clear dirty */
+		raw = GF_CALLOC (sizeof(int), AFR_NUM_CHANGE_LOGS, gf_afr_mt_int32_t);
+		if (!raw)
+			goto err;
 
-static int
-afr_sh_is_innocent (int32_t *array, int child_count)
-{
-        int i   = 0;
-        int ret = 1;   /* innocent until proven guilty */
-
-        for (i = 0; i < child_count; i++) {
-                if (array[i]) {
-                        ret = 0;
-                        break;
-                }
-        }
+		raw[idx] = hton32 (output_dirty[subvol]);
+		ret = dict_set_bin (xattr, AFR_DIRTY, raw,
+				    sizeof(int) * AFR_NUM_CHANGE_LOGS);
+		if (ret)
+			goto err;
+	}
 
-        return ret;
-}
+	/* clear/set pending */
+	for (j = 0; j < priv->child_count; j++) {
+		if (!output_matrix[subvol][j])
+			continue;
 
+		raw = GF_CALLOC (sizeof(int), AFR_NUM_CHANGE_LOGS,
+				 gf_afr_mt_int32_t);
+		if (!raw)
+			goto err;
 
-static int
-afr_sh_is_fool (int32_t *array, int i, int child_count)
-{
-        return array[i];   /* fool if accuses itself */
-}
+		raw[idx] = hton32 (output_matrix[subvol][j]);
 
+		ret = dict_set_bin (xattr, priv->pending_key[j],
+				    raw, sizeof(int) * AFR_NUM_CHANGE_LOGS);
+		if (ret)
+			goto err;
+	}
 
-static int
-afr_sh_is_wise (int32_t *array, int i, int child_count)
-{
-        return !array[i];  /* wise if does not accuse itself */
+	return xattr;
+err:
+	if (xattr)
+		dict_unref (xattr);
+	return NULL;
 }
 
 
-static int
-afr_sh_all_nodes_innocent (afr_node_character *characters,
-                           int child_count)
-{
-        int i   = 0;
-        int ret = 1;
-
-        for (i = 0; i < child_count; i++) {
-                if (characters[i].type != AFR_NODE_INNOCENT) {
-                        ret = 0;
-                        break;
-                }
-        }
-
-        return ret;
+int
+afr_selfheal_undo_pending (call_frame_t *frame, xlator_t *this, inode_t *inode,
+			   unsigned char *sources, unsigned char *sinks,
+			   unsigned char *healed_sinks, afr_transaction_type type,
+			   struct afr_reply *replies, unsigned char *locked_on)
+{
+	afr_private_t *priv = NULL;
+	int i = 0;
+	int j = 0;
+	unsigned char *pending = NULL;
+	int *input_dirty = NULL;
+	int **input_matrix = NULL;
+	int *output_dirty = NULL;
+	int **output_matrix = NULL;
+	dict_t *xattr = NULL;
+
+	priv = this->private;
+
+	pending = alloca0 (priv->child_count);
+
+	input_dirty = alloca0 (priv->child_count * sizeof (int));
+	input_matrix = ALLOC_MATRIX (priv->child_count, int);
+	output_dirty = alloca0 (priv->child_count * sizeof (int));
+	output_matrix = ALLOC_MATRIX (priv->child_count, int);
+
+	afr_selfheal_extract_xattr (this, replies, type, input_dirty,
+				    input_matrix);
+
+	for (i = 0; i < priv->child_count; i++)
+		if (sinks[i] && !healed_sinks[i])
+			pending[i] = 1;
+
+	for (i = 0; i < priv->child_count; i++) {
+		for (j = 0; j < priv->child_count; j++) {
+			if (pending[j])
+				output_matrix[i][j] = 1;
+			else
+				output_matrix[i][j] = -input_matrix[i][j];
+		}
+	}
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!pending[i])
+			output_dirty[i] = -input_dirty[i];
+	}
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!locked_on[i])
+			/* perform post-op only on subvols we had locked
+			   and inspected on.
+			*/
+			continue;
+
+		xattr = afr_selfheal_output_xattr (this, type, output_dirty,
+						   output_matrix, i);
+		if (!xattr) {
+			gf_log (this->name, GF_LOG_ERROR,
+				"unable to allocate xdata for subvol %d", i);
+			continue;
+		}
+
+		afr_selfheal_post_op (frame, this, inode, i, xattr);
+
+		dict_unref (xattr);
+	}
+
+	return 0;
 }
 
 
-static int
-afr_sh_wise_nodes_exist (afr_node_character *characters, int child_count)
-{
-        int i   = 0;
-        int ret = 0;
-
-        for (i = 0; i < child_count; i++) {
-                if (characters[i].type == AFR_NODE_WISE) {
-                        ret = 1;
-                        break;
-                }
-        }
-
-        return ret;
+void
+afr_replies_copy (struct afr_reply *dst, struct afr_reply *src, int count)
+{
+	int i = 0;
+	dict_t *xdata = NULL;
+
+	if (dst == src)
+		return;
+
+	for (i = 0; i < count; i++) {
+		dst[i].valid = src[i].valid;
+		dst[i].op_ret = src[i].op_ret;
+		dst[i].op_errno = src[i].op_errno;
+		dst[i].prestat = src[i].prestat;
+		dst[i].poststat = src[i].poststat;
+		dst[i].preparent = src[i].preparent;
+		dst[i].postparent = src[i].postparent;
+		dst[i].preparent2 = src[i].preparent2;
+		dst[i].postparent2 = src[i].postparent2;
+		if (src[i].xdata)
+			xdata = dict_ref (src[i].xdata);
+		else
+			xdata = NULL;
+		if (dst[i].xdata)
+			dict_unref (dst[i].xdata);
+		dst[i].xdata = xdata;
+		memcpy (dst[i].checksum, src[i].checksum,
+			MD5_DIGEST_LENGTH);
+	}
 }
 
 
-/*
- * The 'wisdom' of a wise node is 0 if any other wise node accuses it.
- * It is 1 if no other wise node accuses it.
- * Only wise nodes with wisdom 1 are sources.
- *
- * If no nodes with wisdom 1 exist, a split-brain has occurred.
- */
-
-static void
-afr_sh_compute_wisdom (int32_t *pending_matrix[],
-                       afr_node_character characters[], int child_count)
+int
+afr_selfheal_fill_dirty (xlator_t *this, int *dirty, int subvol,
+			 int idx, dict_t *xdata)
 {
-        int i = 0;
-        int j = 0;
-
-        for (i = 0; i < child_count; i++) {
-                if (characters[i].type == AFR_NODE_WISE) {
-                        characters[i].wisdom = 1;
-
-                        for (j = 0; j < child_count; j++) {
-                                if ((characters[j].type == AFR_NODE_WISE)
-                                    && pending_matrix[j][i]) {
-
-                                        characters[i].wisdom = 0;
-                                }
-                        }
-                }
-        }
-}
+	void *pending_raw = NULL;
+	int pending[3] = {0, };
 
+	if (dict_get_ptr (xdata, AFR_DIRTY, &pending_raw))
+		return -1;
 
-static int
-afr_sh_wise_nodes_conflict (afr_node_character *characters,
-                            int child_count)
-{
-        int i   = 0;
-        int ret = 1;
+	if (!pending_raw)
+		return -1;
 
-        for (i = 0; i < child_count; i++) {
-                if ((characters[i].type == AFR_NODE_WISE)
-                    && characters[i].wisdom == 1) {
+	memcpy (pending, pending_raw, sizeof(pending));
 
-                        /* There is atleast one bona-fide wise node */
-                        ret = 0;
-                        break;
-                }
-        }
+	dirty[subvol] = ntoh32 (pending[idx]);
 
-        return ret;
+	return 0;
 }
 
 
-static int
-afr_sh_mark_wisest_as_sources (int sources[],
-                               afr_node_character *characters,
-                               int child_count)
-{
-        int nsources = 0;
-        int i        = 0;
-
-        for (i = 0; i < child_count; i++) {
-                if (characters[i].wisdom == 1) {
-                        sources[i] = 1;
-                        nsources++;
-                }
-        }
-
-        return nsources;
-}
-
-static void
-afr_compute_witness_of_fools (int32_t *witnesses, int32_t **pending_matrix,
-                              afr_node_character *characters,
-                              int32_t child_count)
+int
+afr_selfheal_fill_matrix (xlator_t *this, int **matrix, int subvol,
+			  int idx, dict_t *xdata)
 {
-        int i       = 0;
-        int j       = 0;
-        int witness = 0;
-
-        GF_ASSERT (witnesses);
-        GF_ASSERT (pending_matrix);
-        GF_ASSERT (characters);
-        GF_ASSERT (child_count > 0);
-
-        for (i = 0; i < child_count; i++) {
-                if (characters[i].type != AFR_NODE_FOOL)
-                        continue;
-
-                witness = 0;
-                for (j = 0; j < child_count; j++) {
-                        if (i == j)
-                                continue;
-                        witness += pending_matrix[i][j];
-                }
-                witnesses[i] = witness;
-        }
-}
+	int i = 0;
+	void *pending_raw = NULL;
+	int pending[3] = {0, };
+	afr_private_t *priv = NULL;
 
-static int32_t
-afr_find_biggest_witness_among_fools (int32_t *witnesses,
-                                      afr_node_character *characters,
-                                      int32_t child_count)
-{
-        int i               = 0;
-        int biggest_witness = -1;
+	priv = this->private;
 
-        GF_ASSERT (witnesses);
-        GF_ASSERT (characters);
-        GF_ASSERT (child_count > 0);
+	for (i = 0; i < priv->child_count; i++) {
+		if (dict_get_ptr (xdata, priv->pending_key[i], &pending_raw))
+			continue;
 
-        for (i = 0; i < child_count; i++) {
-                if (characters[i].type != AFR_NODE_FOOL)
-                        continue;
+		if (!pending_raw)
+			continue;
 
-                if (biggest_witness < witnesses[i])
-                        biggest_witness = witnesses[i];
-        }
-        return biggest_witness;
-}
+		memcpy (pending, pending_raw, sizeof(pending));
 
-int
-afr_mark_fool_as_source_by_witness (int32_t *sources, int32_t *witnesses,
-                                    afr_node_character *characters,
-                                    int32_t child_count, int32_t witness)
-{
-        int i        = 0;
-        int nsources = 0;
-
-        GF_ASSERT (sources);
-        GF_ASSERT (witnesses);
-        GF_ASSERT (characters);
-        GF_ASSERT (child_count > 0);
-
-        for (i = 0; i < child_count; i++) {
-                if (characters[i].type != AFR_NODE_FOOL)
-                        continue;
-
-                if (witness == witnesses[i]) {
-                        sources[i] = 1;
-                        nsources++;
-                }
-        }
-        return nsources;
-}
+		matrix[subvol][i] = ntoh32 (pending[idx]);
+	}
 
-static int
-afr_mark_biggest_of_fools_as_source (int32_t *sources, int32_t **pending_matrix,
-                                     afr_node_character *characters,
-                                     int child_count)
-{
-        int32_t       biggest_witness = 0;
-        int           nsources        = 0;
-        int32_t       *witnesses      = NULL;
-
-        GF_ASSERT (child_count > 0);
-
-        witnesses = GF_CALLOC (child_count, sizeof (*witnesses),
-                               gf_afr_mt_int32_t);
-        if (NULL == witnesses) {
-                nsources = -1;
-                goto out;
-        }
-
-        afr_compute_witness_of_fools (witnesses, pending_matrix, characters,
-                                      child_count);
-        biggest_witness = afr_find_biggest_witness_among_fools (witnesses,
-                                                                characters,
-                                                                child_count);
-        nsources = afr_mark_fool_as_source_by_witness (sources, witnesses,
-                                                       characters, child_count,
-                                                       biggest_witness);
-out:
-        if (witnesses)
-                GF_FREE (witnesses);
-        return nsources;
+	return 0;
 }
 
-int
-afr_mark_child_as_source_by_uid (int32_t *sources, struct iatt *bufs,
-                                 int32_t *success_children,
-                                 unsigned int child_count, uint32_t uid)
-{
-        int     i        = 0;
-        int     nsources = 0;
-        int     child    = 0;
-
-        for (i = 0; i < child_count; i++) {
-                if (-1 == success_children[i])
-                        break;
-
-                child = success_children[i];
-                if (uid == bufs[child].ia_uid) {
-                        sources[child] = 1;
-                        nsources++;
-                }
-        }
-        return nsources;
-}
 
 int
-afr_get_child_with_lowest_uid (struct iatt *bufs, int32_t *success_children,
-                               unsigned int child_count)
+afr_selfheal_extract_xattr (xlator_t *this, struct afr_reply *replies,
+			    afr_transaction_type type, int *dirty, int **matrix)
 {
-        int     i        = 0;
-        int     smallest = -1;
-        int     child    = 0;
-
-        for (i = 0; i < child_count; i++) {
-                if (-1 == success_children[i])
-                        break;
-                child = success_children[i];
-                if ((smallest == -1) ||
-                    (bufs[child].ia_uid < bufs[smallest].ia_uid)) {
-                        smallest = child;
-                }
-        }
-        return smallest;
-}
+	afr_private_t *priv = NULL;
+	int i = 0;
+	dict_t *xdata = NULL;
+	int idx = -1;
 
-static int
-afr_sh_mark_lowest_uid_as_source (struct iatt *bufs, int32_t *success_children,
-                                  int child_count, int32_t *sources)
-{
-        int   nsources              = 0;
-        int   smallest              = 0;
-
-        smallest = afr_get_child_with_lowest_uid (bufs, success_children,
-                                                  child_count);
-        if (smallest < 0) {
-                nsources = -1;
-                goto out;
-        }
-        nsources = afr_mark_child_as_source_by_uid (sources, bufs,
-                                                    success_children, child_count,
-                                                    bufs[smallest].ia_uid);
-out:
-        return nsources;
-}
-
-int
-afr_get_no_xattr_dir_read_child (xlator_t *this, int32_t *success_children,
-                                 struct iatt *bufs)
-{
-        afr_private_t *priv = NULL;
-        int            i = 0;
-        int            child = -1;
-        int            read_child = -1;
-
-        priv = this->private;
-        for (i = 0; i < priv->child_count; i++) {
-                child = success_children[i];
-                if (child < 0)
-                        break;
-                if (read_child < 0)
-                        read_child = child;
-                else if (bufs[read_child].ia_size < bufs[child].ia_size)
-                        read_child = child;
-        }
-        return read_child;
-}
+	idx = afr_index_for_transaction_type (type);
 
-int
-afr_sh_mark_zero_size_file_as_sink (struct iatt *bufs, int32_t *success_children,
-                                    int child_count, int32_t *sources)
-{
-        int             nsources = 0;
-        int             i = 0;
-        int             child = 0;
-        gf_boolean_t    sink_exists = _gf_false;
-        gf_boolean_t    source_exists = _gf_false;
-        int             source = -1;
-
-        for (i = 0; i < child_count; i++) {
-                child = success_children[i];
-                if (child < 0)
-                        break;
-                if (!bufs[child].ia_size) {
-                        sink_exists = _gf_true;
-                        continue;
-                }
-                if (!source_exists) {
-                        source_exists = _gf_true;
-                        source = child;
-                        continue;
-                }
-                if (bufs[source].ia_size != bufs[child].ia_size) {
-                        nsources = -1;
-                        goto out;
-                }
-        }
-        if (!source_exists && !sink_exists) {
-                nsources = -1;
-                goto out;
-        }
-
-        if (!source_exists || !sink_exists)
-                goto out;
-
-        for (i = 0; i < child_count; i++) {
-                child = success_children[i];
-                if (child < 0)
-                        break;
-                if (bufs[child].ia_size) {
-                        sources[child] = 1;
-                        nsources++;
-                }
-        }
-out:
-        return nsources;
-}
+	priv = this->private;
 
-char *
-afr_get_character_str (afr_node_type type)
-{
-        char *character = NULL;
-
-        switch (type) {
-        case AFR_NODE_INNOCENT:
-                character = "innocent";
-                break;
-        case AFR_NODE_FOOL:
-                character = "fool";
-                break;
-        case AFR_NODE_WISE:
-                character = "wise";
-                break;
-        default:
-                character = "invalid";
-                break;
-        }
-        return character;
-}
+	for (i = 0; i < priv->child_count; i++) {
+		if (!replies[i].xdata)
+			continue;
 
-afr_node_type
-afr_find_child_character_type (int32_t *pending_row, int32_t child,
-                               unsigned int child_count)
-{
-        afr_node_type type = AFR_NODE_INVALID;
+		xdata = replies[i].xdata;
 
-        GF_ASSERT ((child >= 0) && (child < child_count));
+		afr_selfheal_fill_dirty (this, dirty, i, idx, xdata);
+		afr_selfheal_fill_matrix (this, matrix, i, idx, xdata);
+	}
 
-        if (afr_sh_is_innocent (pending_row, child_count))
-                type = AFR_NODE_INNOCENT;
-        else if (afr_sh_is_fool (pending_row, child, child_count))
-                type = AFR_NODE_FOOL;
-        else if (afr_sh_is_wise (pending_row, child, child_count))
-                type = AFR_NODE_WISE;
-        return type;
+	return 0;
 }
 
-int
-afr_build_sources (xlator_t *this, dict_t **xattr, struct iatt *bufs,
-                   int32_t **pending_matrix, int32_t *sources,
-                   int32_t *success_children, afr_transaction_type type,
-                   int32_t *subvol_status, gf_boolean_t ignore_ignorant)
-{
-        afr_private_t           *priv = NULL;
-        afr_self_heal_type      sh_type    = AFR_SELF_HEAL_INVALID;
-        int                     nsources   = -1;
-        unsigned char           *ignorant_subvols = NULL;
-        unsigned int            child_count = 0;
-
-        priv = this->private;
-        child_count = priv->child_count;
-
-        if (afr_get_children_count (success_children, priv->child_count) == 0)
-                goto out;
-
-        if (!ignore_ignorant) {
-                ignorant_subvols = GF_CALLOC (sizeof (*ignorant_subvols),
-                                              child_count, gf_afr_mt_char);
-                if (NULL == ignorant_subvols)
-                        goto out;
-        }
-
-        afr_build_pending_matrix (priv->pending_key, pending_matrix,
-                                  ignorant_subvols, xattr, type,
-                                  priv->child_count);
-
-        if (!ignore_ignorant)
-                afr_mark_ignorant_subvols_as_pending (pending_matrix,
-                                                      ignorant_subvols,
-                                                      priv->child_count);
-        sh_type = afr_self_heal_type_for_transaction (type);
-        if (AFR_SELF_HEAL_INVALID == sh_type)
-                goto out;
-
-        afr_sh_print_pending_matrix (pending_matrix, this);
-
-        nsources = afr_mark_sources (this, sources, pending_matrix, bufs,
-                                     sh_type, success_children, subvol_status);
-out:
-        GF_FREE (ignorant_subvols);
-        return nsources;
-}
 
-void
-afr_find_character_types (afr_node_character *characters,
-                          int32_t **pending_matrix, int32_t *success_children,
-                          unsigned int child_count)
-{
-        afr_node_type type  = AFR_NODE_INVALID;
-        int           child = 0;
-        int           i     = 0;
-
-        for (i = 0; i < child_count; i++) {
-                child = success_children[i];
-                if (child == -1)
-                        break;
-                type = afr_find_child_character_type (pending_matrix[child],
-                                                      child, child_count);
-                characters[child].type = type;
-        }
-}
 
-void
-afr_mark_success_children_sources (int32_t *sources, int32_t *success_children,
-                                   unsigned int child_count)
-{
-        int i = 0;
-        for (i = 0; i < child_count; i++) {
-                if (success_children[i] == -1)
-                        break;
-                sources[success_children[i]] = 1;
-        }
-}
-/**
- * mark_sources: Mark all 'source' nodes and return number of source
- * nodes found
+/*
+ * This function determines if a self-heal is required for a given inode,
+ * and if needed, in what direction.
+ *
+ * locked_on[] is the array representing servers which have been locked and
+ * from which xattrs have been fetched for analysis.
  *
- * A node (a row in the pending matrix) belongs to one of
- * three categories:
+ * The output of the function is by filling the arrays sources[] and sinks[].
  *
- * M is the pending matrix.
+ * sources[i] is set if i'th server is an eligible source for a selfheal.
  *
- * 'innocent' - M[i] is all zeroes
- * 'fool'     - M[i] has i'th element = 1 (self-reference)
- * 'wise'     - M[i] has i'th element = 0, others are 1 or 0.
+ * sinks[i] is set if i'th server needs to be healed.
  *
- * All 'innocent' nodes are sinks. If all nodes are innocent, no self-heal is
- * needed.
+ * if sources[0..N] are all set, there is no need for a selfheal.
  *
- * A 'wise' node can be a source. If two 'wise' nodes conflict, it is
- * a split-brain. If one wise node refers to the other but the other doesn't
- * refer back, the referrer is a source.
+ * if sinks[0..N] are all set, the inode is in split brain.
  *
- * All fools are sinks, unless there are no 'wise' nodes. In that case,
- * one of the fools is made a source.
  */
 
 int
-afr_mark_sources (xlator_t *this, int32_t *sources, int32_t **pending_matrix,
-                  struct iatt *bufs, afr_self_heal_type type,
-                  int32_t *success_children, int32_t *subvol_status)
-{
-        /* stores the 'characters' (innocent, fool, wise) of the nodes */
-        afr_node_character *characters =  NULL;
-        int                nsources    = -1;
-        unsigned int       child_count = 0;
-        afr_private_t      *priv       = NULL;
-
-        priv = this->private;
-        child_count = priv->child_count;
-        characters = GF_CALLOC (sizeof (afr_node_character),
-                                child_count, gf_afr_mt_afr_node_character);
-        if (!characters)
-                goto out;
-
-        this = THIS;
-
-        /* start clean */
-        memset (sources, 0, sizeof (*sources) * child_count);
-        nsources = 0;
-        afr_find_character_types (characters, pending_matrix, success_children,
-                                  child_count);
-        if (afr_sh_all_nodes_innocent (characters, child_count)) {
-                switch (type) {
-                case AFR_SELF_HEAL_METADATA:
-                        nsources = afr_sh_mark_lowest_uid_as_source (bufs,
-                                                             success_children,
-                                                             child_count,
-                                                             sources);
-                        break;
-                case AFR_SELF_HEAL_DATA:
-                        nsources = afr_sh_mark_zero_size_file_as_sink (bufs,
-                                                             success_children,
-                                                             child_count,
-                                                             sources);
-                        if ((nsources < 0) && subvol_status)
-                                *subvol_status |= SPLIT_BRAIN;
-                        break;
-                default:
-                        break;
-                }
-                goto out;
-        }
-
-        if (afr_sh_wise_nodes_exist (characters, child_count)) {
-                afr_sh_compute_wisdom (pending_matrix, characters, child_count);
-
-                if (afr_sh_wise_nodes_conflict (characters, child_count)) {
-                        if (subvol_status)
-                                *subvol_status |= SPLIT_BRAIN;
-                        nsources = -1;
-                } else {
-                        nsources = afr_sh_mark_wisest_as_sources (sources,
-                                                                  characters,
-                                                                  child_count);
-                }
-        } else {
-                if (subvol_status)
-                        *subvol_status |= ALL_FOOLS;
-                nsources = afr_mark_biggest_of_fools_as_source (sources,
-                                                                pending_matrix,
-                                                                characters,
-                                                                child_count);
-        }
-
-out:
-        if (nsources == 0)
-                afr_mark_success_children_sources (sources, success_children,
-                                                   child_count);
-        GF_FREE (characters);
-
-        gf_log (this->name, GF_LOG_DEBUG, "Number of sources: %d", nsources);
-        return nsources;
-}
-
-void
-afr_sh_pending_to_delta (afr_private_t *priv, dict_t **xattr,
-                         int32_t *delta_matrix[], unsigned char success[],
-                         int child_count, afr_transaction_type type)
-{
-        int i = 0;
-        int j = 0;
-
-        afr_build_pending_matrix (priv->pending_key, delta_matrix, NULL,
-                                  xattr, type, priv->child_count);
-        for (i = 0; i < priv->child_count; i++)
-                for (j = 0; j < priv->child_count; j++)
-                        delta_matrix[i][j] = -delta_matrix[i][j];
+afr_selfheal_find_direction (call_frame_t *frame, xlator_t *this,
+			     struct afr_reply *replies,
+			     afr_transaction_type type, unsigned char *locked_on,
+			     unsigned char *sources, unsigned char *sinks)
+{
+	afr_private_t *priv = NULL;
+	int i = 0;
+	int j = 0;
+	int *dirty = NULL;
+	int **matrix = NULL;
+	char *accused = NULL;
+
+	priv = this->private;
+
+	dirty = alloca0 (priv->child_count * sizeof (int));
+	accused = alloca0 (priv->child_count);
+	matrix = ALLOC_MATRIX(priv->child_count, int);
+
+	/* First construct the pending matrix for further analysis */
+	afr_selfheal_extract_xattr (this, replies, type, dirty, matrix);
+
+	/* Next short list all accused to exclude them from being sources */
+	for (i = 0; i < priv->child_count; i++) {
+		for (j = 0; j < priv->child_count; j++) {
+			if (matrix[i][j])
+				accused[j] = 1;
+		}
+	}
+
+	/* Short list all non-accused as sources */
+	memset (sources, 0, priv->child_count);
+	for (i = 0; i < priv->child_count; i++) {
+		if (!accused[i] && locked_on[i])
+			sources[i] = 1;
+	}
+
+	/* Everyone accused by sources are sinks */
+	memset (sinks, 0, priv->child_count);
+	for (i = 0; i < priv->child_count; i++) {
+		if (!sources[i])
+			continue;
+		for (j = 0; j < priv->child_count; j++) {
+			if (matrix[i][j])
+				sinks[j] = 1;
+		}
+	}
+
+	/* If any source has 'dirty' bit, pick first
+	   'dirty' source and make everybody else sinks */
+	for (i = 0; i < priv->child_count; i++) {
+		if (sources[i] && dirty[i]) {
+			for (j = 0; j < priv->child_count; j++) {
+				if (j != i) {
+					sources[j] = 0;
+					sinks[j] = 1;
+				}
+			}
+			break;
+		}
+	}
+
+	/* If no sources, all locked nodes are sinks - split brain */
+	if (AFR_COUNT (sources, priv->child_count) == 0) {
+		for (i = 0; i < priv->child_count; i++) {
+			if (locked_on[i])
+				sinks[i] = 1;
+		}
+	}
+
+	return 0;
 }
 
 
 int
-afr_sh_delta_to_xattr (afr_private_t *priv,
-                       int32_t *delta_matrix[], dict_t *xattr[],
-                       int child_count, afr_transaction_type type)
+afr_selfheal_discover_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+			   int op_ret, int op_errno, inode_t *inode,
+			   struct iatt *buf, dict_t *xdata, struct iatt *parbuf)
 {
-        int      i       = 0;
-        int      j       = 0;
-        int      k       = 0;
-        int      ret     = 0;
-        int32_t *pending = NULL;
-
-        for (i = 0; i < child_count; i++) {
-                if (!xattr[i])
-                        continue;
-
-                for (j = 0; j < child_count; j++) {
-                        pending = GF_CALLOC (sizeof (int32_t), 3,
-                                             gf_afr_mt_int32_t);
-
-                        if (!pending)
-                                continue;
-                        /* 3 = data+metadata+entry */
-
-                        k = afr_index_for_transaction_type (type);
-
-                        pending[k] = hton32 (delta_matrix[i][j]);
-
-                        ret = dict_set_bin (xattr[i], priv->pending_key[j],
-                                            pending,
-                                            3 * sizeof (int32_t));
-                        if (ret < 0)
-                                gf_log (THIS->name, GF_LOG_WARNING,
-                                        "Unable to set dict value.");
-                }
-        }
-        return 0;
-}
+	afr_local_t *local = NULL;
+	int i = -1;
 
+	local = frame->local;
+	i = (long) cookie;
 
-int
-afr_sh_has_metadata_pending (dict_t *xattr, xlator_t *this)
-{
-        /* Indexable by result of afr_index_for_transaction_type(): 0 -- 2. */
-        int32_t        pending[3]  = {0,};
-        void          *pending_raw = NULL;
-        afr_private_t *priv        = NULL;
-        int            ret         = -1;
-        int            i           = 0;
-        int            j           = 0;
+	local->replies[i].valid = 1;
+	local->replies[i].op_ret = op_ret;
+	local->replies[i].op_errno = op_errno;
+	if (buf)
+		local->replies[i].poststat = *buf;
+	if (parbuf)
+		local->replies[i].postparent = *parbuf;
+	if (xdata)
+		local->replies[i].xdata = dict_ref (xdata);
 
-        priv = this->private;
+	syncbarrier_wake (&local->barrier);
 
-        for (i = 0; i < priv->child_count; i++) {
-                ret = dict_get_ptr (xattr, priv->pending_key[i],
-                                    &pending_raw);
-
-                if (ret != 0)
-                        return 0;
+	return 0;
+}
 
-                memcpy (pending, pending_raw, sizeof(pending));
-                j = afr_index_for_transaction_type (AFR_METADATA_TRANSACTION);
 
-                if (pending[j])
-                        return 1;
-        }
+inode_t *
+afr_selfheal_unlocked_lookup_on (call_frame_t *frame, inode_t *parent,
+				 const char *name, struct afr_reply *replies,
+				 unsigned char *lookup_on)
+{
+	loc_t loc = {0, };
+	dict_t *xattr_req = NULL;
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	inode_t *inode = NULL;
 
-        return 0;
-}
+	local = frame->local;
+	priv = frame->this->private;
 
+	xattr_req = dict_new ();
+	if (!xattr_req)
+		return NULL;
 
-int
-afr_sh_has_data_pending (dict_t *xattr, xlator_t *this)
-{
-        /* Indexable by result of afr_index_for_transaction_type(): 0 -- 2. */
-        int32_t        pending[3]  = {0,};
-        void          *pending_raw = NULL;
-        afr_private_t *priv        = NULL;
-        int            ret         = -1;
-        int            i           = 0;
-        int            j           = 0;
+	if (afr_xattr_req_prepare (frame->this, xattr_req) != 0) {
+		dict_destroy (xattr_req);
+		return NULL;
+	}
 
-        priv = this->private;
+	inode = inode_new (parent->table);
+	if (!inode) {
+		dict_destroy (xattr_req);
+		return NULL;
+	}
 
-        for (i = 0; i < priv->child_count; i++) {
-                ret = dict_get_ptr (xattr, priv->pending_key[i],
-                                    &pending_raw);
+	loc.parent = inode_ref (parent);
+	uuid_copy (loc.pargfid, parent->gfid);
+	loc.name = name;
+	loc.inode = inode_ref (inode);
 
-                if (ret != 0)
-                        return 0;
+	AFR_ONLIST (lookup_on, frame, afr_selfheal_discover_cbk, lookup, &loc,
+		    xattr_req);
 
-                memcpy (pending, pending_raw, sizeof(pending));
-                j = afr_index_for_transaction_type (AFR_DATA_TRANSACTION);
+	afr_replies_copy (replies, local->replies, priv->child_count);
 
-                if (pending[j])
-                        return 1;
-        }
+	loc_wipe (&loc);
+	dict_unref (xattr_req);
 
-        return 0;
+	return inode;
 }
 
 
 int
-afr_sh_has_entry_pending (dict_t *xattr, xlator_t *this)
+afr_selfheal_unlocked_discover_on (call_frame_t *frame, inode_t *inode,
+				   uuid_t gfid, struct afr_reply *replies,
+				   unsigned char *discover_on)
 {
-        /* Indexable by result of afr_index_for_transaction_type(): 0 -- 2. */
-        int32_t        pending[3]  = {0,};
-        void          *pending_raw = NULL;
-        afr_private_t *priv        = NULL;
-        int            ret         = -1;
-        int            i           = 0;
-        int            j           = 0;
+	loc_t loc = {0, };
+	dict_t *xattr_req = NULL;
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+
+	local = frame->local;
+	priv = frame->this->private;
 
-        priv = this->private;
+	xattr_req = dict_new ();
+	if (!xattr_req)
+		return -ENOMEM;
 
-        for (i = 0; i < priv->child_count; i++) {
-                ret = dict_get_ptr (xattr, priv->pending_key[i],
-                                    &pending_raw);
+	if (afr_xattr_req_prepare (frame->this, xattr_req) != 0) {
+		dict_destroy (xattr_req);
+		return -ENOMEM;
+	}
 
-                if (ret != 0)
-                        return 0;
+	loc.inode = inode_ref (inode);
+	uuid_copy (loc.gfid, gfid);
 
-                memcpy (pending, pending_raw, sizeof(pending));
-                j = afr_index_for_transaction_type (AFR_ENTRY_TRANSACTION);
+	AFR_ONLIST (discover_on, frame, afr_selfheal_discover_cbk, lookup, &loc,
+		    xattr_req);
 
-                if (pending[j])
-                        return 1;
-        }
+	afr_replies_copy (replies, local->replies, priv->child_count);
 
-        return 0;
+	loc_wipe (&loc);
+	dict_unref (xattr_req);
+
+	return 0;
 }
 
 int
-afr_sh_missing_entries_done (call_frame_t *frame, xlator_t *this)
+afr_selfheal_unlocked_discover (call_frame_t *frame, inode_t *inode,
+				uuid_t gfid, struct afr_reply *replies)
 {
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-
-        local = frame->local;
-        sh = &local->self_heal;
-
-        afr_sh_reset (frame, this);
-
-        if (local->govinda_gOvinda) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "split brain found, aborting selfheal of %s",
-                        local->loc.path);
-                sh->op_failed = 1;
-        }
-
-        if (sh->op_failed) {
-                sh->completion_cbk (frame, this);
-        } else {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "proceeding to metadata check on %s",
-                        local->loc.path);
-                afr_self_heal_metadata (frame, this);
-        }
-
-        return 0;
+	afr_private_t *priv = NULL;
+
+	priv = frame->this->private;
+
+	return afr_selfheal_unlocked_discover_on (frame, inode, gfid, replies,
+						  priv->child_up);
 }
 
 
-static int
-afr_sh_missing_entries_finish (call_frame_t *frame, xlator_t *this)
+int
+afr_selfheal_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		       int op_ret, int op_errno, dict_t *xdata)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
+	afr_local_t *local = NULL;
+	int i = 0;
+
+	local = frame->local;
+	i = (long) cookie;
 
-        local = frame->local;
-        int_lock = &local->internal_lock;
+	local->replies[i].valid = 1;
+	local->replies[i].op_ret = op_ret;
+	local->replies[i].op_errno = op_errno;
 
-        int_lock->lock_cbk = afr_sh_missing_entries_done;
-        afr_unlock (frame, this);
+	syncbarrier_wake (&local->barrier);
 
-        return 0;
+	return 0;
 }
 
+
 int
-afr_sh_common_create (afr_self_heal_t *sh, unsigned int child_count)
+afr_selfheal_locked_fill (call_frame_t *frame, xlator_t *this,
+			  unsigned char *locked_on)
 {
-        int     ret = -ENOMEM;
-        sh->buf = GF_CALLOC (child_count, sizeof (*sh->buf),
-                             gf_afr_mt_iatt);
-        if (!sh->buf)
-                goto out;
-        sh->parentbufs = GF_CALLOC (child_count, sizeof (*sh->parentbufs),
-                                    gf_afr_mt_iatt);
-        if (!sh->parentbufs)
-                goto out;
-        sh->child_errno = GF_CALLOC (child_count, sizeof (*sh->child_errno),
-                                     gf_afr_mt_int);
-        if (!sh->child_errno)
-                goto out;
-        sh->success_children = afr_children_create (child_count);
-        if (!sh->success_children)
-                goto out;
-        sh->fresh_children = afr_children_create (child_count);
-        if (!sh->fresh_children)
-                goto out;
-        sh->xattr = GF_CALLOC (child_count, sizeof (*sh->xattr),
-                               gf_afr_mt_dict_t);
-        if (!sh->xattr)
-                goto out;
-        ret = 0;
-out:
-        return ret;
-}
+	int i = 0;
+	afr_private_t *priv = NULL;
+	afr_local_t *local = NULL;
+	int count = 0;
 
-void
-afr_sh_common_lookup_resp_handler (call_frame_t *frame, void *cookie,
-                                   xlator_t *this,
-                                   int32_t op_ret, int32_t op_errno,
-                                   inode_t *inode, struct iatt *buf,
-                                   dict_t *xattr, struct iatt *postparent,
-                                   loc_t *loc)
-{
-        int              child_index = 0;
-        afr_local_t     *local = NULL;
-        afr_private_t   *priv = NULL;
-        afr_self_heal_t *sh = NULL;
-
-        local = frame->local;
-        priv = this->private;
-        sh   = &local->self_heal;
-        child_index = (long) cookie;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == 0) {
-                        sh->buf[child_index] = *buf;
-                        sh->parentbufs[child_index] = *postparent;
-                        sh->success_children[sh->success_count] = child_index;
-                        sh->success_count++;
-                        sh->xattr[child_index] = dict_ref (xattr);
-                } else {
-                        gf_log (this->name, GF_LOG_DEBUG, "path %s on subvolume"
-                                " %s => -1 (%s)", loc->path,
-                                priv->children[child_index]->name,
-                                strerror (op_errno));
-                        local->self_heal.child_errno[child_index] = op_errno;
-                }
-        }
-        UNLOCK (&frame->lock);
-        return;
-}
+	local = frame->local;
+	priv = this->private;
 
-gf_boolean_t
-afr_valid_ia_type (ia_type_t ia_type)
-{
-        switch (ia_type) {
-        case IA_IFSOCK:
-        case IA_IFREG:
-        case IA_IFBLK:
-        case IA_IFCHR:
-        case IA_IFIFO:
-        case IA_IFLNK:
-        case IA_IFDIR:
-                return _gf_true;
-        default:
-                return _gf_false;
-        }
-        return _gf_false;
-}
+	for (i = 0; i < priv->child_count; i++) {
+		if (local->replies[i].valid && local->replies[i].op_ret == 0) {
+			locked_on[i] = 1;
+			count++;
+		} else {
+			locked_on[i] = 0;
+		}
+	}
 
-int
-afr_impunge_frame_create (call_frame_t *frame, xlator_t *this,
-                          int active_source, call_frame_t **impunge_frame)
-{
-        afr_local_t     *local         = NULL;
-        afr_local_t     *impunge_local = NULL;
-        afr_self_heal_t *impunge_sh    = NULL;
-        int32_t         op_errno       = 0;
-        afr_private_t   *priv          = NULL;
-        int             ret            = 0;
-        call_frame_t    *new_frame     = NULL;
-
-        op_errno = ENOMEM;
-        priv = this->private;
-        new_frame = copy_frame (frame);
-        if (!new_frame) {
-                goto out;
-        }
-
-        AFR_LOCAL_ALLOC_OR_GOTO (impunge_local, out);
-
-        local = frame->local;
-        new_frame->local = impunge_local;
-        impunge_sh = &impunge_local->self_heal;
-        impunge_sh->sh_frame = frame;
-        impunge_sh->active_source = active_source;
-        impunge_local->child_up  = memdup (local->child_up,
-                                           sizeof (*local->child_up) *
-                                           priv->child_count);
-        if (!impunge_local->child_up)
-                goto out;
-
-        impunge_local->pending = afr_matrix_create (priv->child_count,
-                                                    AFR_NUM_CHANGE_LOGS);
-        if (!impunge_local->pending)
-                goto out;
-
-        ret = afr_sh_common_create (impunge_sh, priv->child_count);
-        if (ret) {
-                op_errno = -ret;
-                goto out;
-        }
-        op_errno = 0;
-        *impunge_frame = new_frame;
-out:
-        if (op_errno && new_frame)
-                AFR_STACK_DESTROY (new_frame);
-        return -op_errno;
+	return count;
 }
 
-void
-afr_sh_missing_entry_call_impunge_recreate (call_frame_t *frame, xlator_t *this,
-                                            struct iatt *buf,
-                                            struct iatt *postparent,
-                                            afr_impunge_done_cbk_t impunge_done)
-{
-        call_frame_t    *impunge_frame = NULL;
-        afr_local_t     *local = NULL;
-        afr_local_t     *impunge_local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_self_heal_t *impunge_sh = NULL;
-        int             ret = 0;
-        unsigned int    enoent_count = 0;
-        afr_private_t   *priv = NULL;
-        int             i = 0;
-        int32_t         op_errno = 0;
-
-        local = frame->local;
-        sh    = &local->self_heal;
-        priv  = this->private;
-
-        enoent_count = afr_errno_count (NULL, sh->child_errno,
-                                        priv->child_count, ENOENT);
-        if (!enoent_count) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "no missing files - %s. proceeding to metadata check",
-                        local->loc.path);
-                goto out;
-        }
-        sh->impunge_done = impunge_done;
-        ret = afr_impunge_frame_create (frame, this, sh->source, &impunge_frame);
-        if (ret)
-                goto out;
-        impunge_local = impunge_frame->local;
-        impunge_sh    = &impunge_local->self_heal;
-        loc_copy (&impunge_local->loc, &local->loc);
-        ret = afr_build_parent_loc (&impunge_sh->parent_loc,
-                                    &impunge_local->loc, &op_errno);
-        if (ret) {
-                ret = -op_errno;
-                goto out;
-        }
-        impunge_local->call_count = enoent_count;
-        impunge_sh->entrybuf = sh->buf[sh->source];
-        impunge_sh->parentbuf = sh->parentbufs[sh->source];
-        for (i = 0; i < priv->child_count; i++) {
-                if (!impunge_local->child_up[i]) {
-                        impunge_sh->child_errno[i] = ENOTCONN;
-                        continue;
-                }
-                if (sh->child_errno[i] != ENOENT) {
-                        impunge_sh->child_errno[i] = EEXIST;
-                        continue;
-                }
-        }
-        for (i = 0; i < priv->child_count; i++) {
-                if (sh->child_errno[i] != ENOENT)
-                        continue;
-                afr_sh_entry_impunge_create (impunge_frame, this, i);
-                enoent_count--;
-        }
-        GF_ASSERT (!enoent_count);
-        return;
-out:
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "impunge of %s failed, "
-                        "reason: %s", local->loc.path, strerror (-ret));
-                sh->op_failed = 1;
-        }
-        afr_sh_missing_entries_finish (frame, this);
-}
 
 int
-afr_sh_create_entry_cbk (call_frame_t *frame, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno)
+afr_selfheal_tryinodelk (call_frame_t *frame, xlator_t *this, inode_t *inode,
+			 char *dom, off_t off, size_t size,
+			 unsigned char *locked_on)
 {
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        if (op_ret < 0)
-                sh->op_failed = 1;
-        afr_sh_missing_entries_finish (frame, this);
-        return 0;
-}
+	loc_t loc = {0,};
+	struct gf_flock flock = {0, };
 
-static int
-sh_missing_entries_create (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        int              type = 0;
-        struct iatt     *buf = NULL;
-        struct iatt     *postparent = NULL;
-
-        local = frame->local;
-        sh = &local->self_heal;
-
-        buf = &sh->buf[sh->source];
-        postparent = &sh->parentbufs[sh->source];
-
-        type = buf->ia_type;
-        if (!afr_valid_ia_type (type)) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s: unknown file type: 0%o", local->loc.path, type);
-                local->govinda_gOvinda = 1;
-                afr_sh_missing_entries_finish (frame, this);
-                goto out;
-        }
-
-        afr_sh_missing_entry_call_impunge_recreate (frame, this,
-                                                    buf, postparent,
-                                                    afr_sh_create_entry_cbk);
-out:
-        return 0;
-}
+	loc.inode = inode_ref (inode);
+	uuid_copy (loc.gfid, inode->gfid);
 
-void
-afr_sh_missing_entries_lookup_done (call_frame_t *frame, xlator_t *this,
-                                    int32_t op_ret, int32_t op_errno)
-{
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        ia_type_t       ia_type = IA_INVAL;
-        int32_t         nsources = 0;
-        loc_t           *loc = NULL;
-        int32_t         subvol_status = 0;
-        afr_transaction_type txn_type = AFR_DATA_TRANSACTION;
-        gf_boolean_t    split_brain = _gf_false;
-        int             read_child = -1;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-        loc = &local->loc;
-
-        if (op_ret < 0) {
-                if (op_errno == EIO)
-                        local->govinda_gOvinda = 1;
-                // EIO can happen if finding the fresh parent dir failed
-                goto out;
-        }
-
-        //now No chance for the ia_type to conflict
-        ia_type = sh->buf[sh->success_children[0]].ia_type;
-        txn_type = afr_transaction_type_get (ia_type);
-        nsources = afr_build_sources (this, sh->xattr, sh->buf,
-                                      sh->pending_matrix, sh->sources,
-                                      sh->success_children, txn_type,
-                                      &subvol_status, _gf_false);
-        if (nsources < 0) {
-                gf_log (this->name, GF_LOG_INFO, "No sources for dir of %s,"
-                        " in missing entry self-heal, continuing with the rest"
-                        " of the self-heals", local->loc.path);
-                if (subvol_status & SPLIT_BRAIN) {
-                        split_brain = _gf_true;
-                        switch (txn_type) {
-                        case AFR_DATA_TRANSACTION:
-                                nsources = 1;
-                                sh->sources[sh->success_children[0]] = 1;
-                                break;
-                        case AFR_ENTRY_TRANSACTION:
-                                read_child = afr_get_no_xattr_dir_read_child
-                                                          (this,
-                                                           sh->success_children,
-                                                           sh->buf);
-                                sh->sources[read_child] = 1;
-                                nsources = 1;
-                                break;
-                        default:
-                                op_errno = EIO;
-                                goto out;
-                        }
-                } else {
-                        op_errno = EIO;
-                        goto out;
-                }
-        }
-
-        afr_get_fresh_children (sh->success_children, sh->sources,
-                                sh->fresh_children, priv->child_count);
-        sh->source = sh->fresh_children[0];
-        if (sh->source == -1) {
-                gf_log (this->name, GF_LOG_DEBUG, "No active sources found.");
-                op_errno = EIO;
-                goto out;
-        }
-
-        if (sh->gfid_sh_success_cbk)
-                sh->gfid_sh_success_cbk (frame, this);
-        sh->type = sh->buf[sh->source].ia_type;
-        if (uuid_is_null (loc->inode->gfid))
-                uuid_copy (loc->gfid, sh->buf[sh->source].ia_gfid);
-        if (split_brain) {
-                afr_sh_missing_entries_finish (frame, this);
-        } else {
-                sh_missing_entries_create (frame, this);
-        }
-        return;
-out:
-        sh->op_failed = 1;
-        afr_sh_set_error (sh, op_errno);
-        afr_sh_missing_entries_finish (frame, this);
-        return;
-}
+	flock.l_type = F_WRLCK;
+	flock.l_start = off;
+	flock.l_len = size;
 
-static int
-afr_sh_common_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, inode_t *inode,
-                          struct iatt *buf, dict_t *xattr,
-                          struct iatt *postparent)
-{
-        int                     call_count = 0;
-        afr_local_t             *local = NULL;
-        afr_self_heal_t         *sh    = NULL;
-        afr_private_t           *priv  = NULL;
-
-        local = frame->local;
-        sh    = &local->self_heal;
-        priv  = this->private;
-
-        afr_sh_common_lookup_resp_handler (frame, cookie, this, op_ret,
-                                           op_errno, inode, buf, xattr,
-                                           postparent, &sh->lookup_loc);
-        call_count = afr_frame_return (frame);
-
-        if (call_count)
-                goto out;
-        op_ret = -1;
-        if (!sh->success_count) {
-                op_errno = afr_resultant_errno_get (NULL, sh->child_errno,
-                                                    priv->child_count);
-                gf_log (this->name, GF_LOG_ERROR, "Failed to lookup %s, "
-                        "reason %s", sh->lookup_loc.path,
-                        strerror (op_errno));
-                goto done;
-        }
-
-        if ((sh->lookup_flags & AFR_LOOKUP_FAIL_CONFLICTS) &&
-            (afr_conflicting_iattrs (sh->buf, sh->success_children,
-                                     priv->child_count,
-                                     sh->lookup_loc.path, this->name))) {
-                op_errno = EIO;
-                gf_log (this->name, GF_LOG_ERROR, "Conflicting entries "
-                        "for %s", sh->lookup_loc.path);
-                goto done;
-        }
-
-        if ((sh->lookup_flags & AFR_LOOKUP_FAIL_MISSING_GFIDS) &&
-            (afr_gfid_missing_count (this->name, sh->success_children,
-                                     sh->buf, priv->child_count,
-                                     sh->lookup_loc.path))) {
-                op_errno = ENODATA;
-                gf_log (this->name, GF_LOG_ERROR, "Missing Gfids "
-                        "for %s", sh->lookup_loc.path);
-                goto done;
-        }
-        op_ret = 0;
-
-done:
-        sh->lookup_done (frame, this, op_ret, op_errno);
-out:
-        return 0;
-}
+	AFR_ONALL (frame, afr_selfheal_lock_cbk, inodelk, dom,
+		    &loc, F_SETLK, &flock, NULL);
 
-int
-afr_sh_remove_entry_cbk (call_frame_t *frame, xlator_t *this, int child,
-                         int32_t op_ret, int32_t op_errno)
-{
-        int             call_count = 0;
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-
-        local = frame->local;
-        sh = &local->self_heal;
-
-        GF_ASSERT (sh->post_remove_call);
-        if ((op_ret == -1) && (op_errno != ENOENT)) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "purge entry %s failed, on child %d reason, %s",
-                        local->loc.path, child, strerror (op_errno));
-                LOCK (&frame->lock);
-                {
-                        afr_sh_set_error (sh, EIO);
-                        sh->op_failed = 1;
-                }
-                UNLOCK (&frame->lock);
-        }
-        call_count = afr_frame_return (frame);
-        if (call_count == 0)
-                sh->post_remove_call (frame, this);
-        return 0;
-}
+	loc_wipe (&loc);
 
-void
-afr_sh_call_entry_expunge_remove (call_frame_t *frame, xlator_t *this,
-                                  int child_index, struct iatt *buf,
-                                  struct iatt *parentbuf,
-                                  afr_expunge_done_cbk_t expunge_done)
-{
-        call_frame_t    *expunge_frame = NULL;
-        afr_local_t     *local = NULL;
-        afr_local_t     *expunge_local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_self_heal_t *expunge_sh = NULL;
-        int32_t         op_errno = 0;
-        int             ret = 0;
-
-        expunge_frame = copy_frame (frame);
-        if (!expunge_frame) {
-                goto out;
-        }
-
-        AFR_LOCAL_ALLOC_OR_GOTO (expunge_local, out);
-
-        local = frame->local;
-        sh = &local->self_heal;
-        expunge_frame->local = expunge_local;
-        expunge_sh = &expunge_local->self_heal;
-        expunge_sh->sh_frame = frame;
-        loc_copy (&expunge_local->loc, &local->loc);
-        ret = afr_build_parent_loc (&expunge_sh->parent_loc,
-                                    &expunge_local->loc, &op_errno);
-        if (ret) {
-                ret = -op_errno;
-                goto out;
-        }
-        sh->expunge_done = expunge_done;
-        afr_sh_entry_expunge_remove (expunge_frame, this, child_index, buf,
-                                     parentbuf);
-        return;
-out:
-        gf_log (this->name, GF_LOG_ERROR, "Expunge of %s failed, reason: %s",
-                local->loc.path, strerror (op_errno));
-        expunge_done (frame, this, child_index, -1, op_errno);
+	return afr_selfheal_locked_fill (frame, this, locked_on);
 }
 
-void
-afr_sh_remove_stale_lookup_info (afr_self_heal_t *sh, int32_t *success_children,
-                                 int32_t *fresh_children,
-                                 unsigned int child_count)
-{
-        int     i = 0;
-
-        for (i = 0; i < child_count; i++) {
-                if (afr_is_child_present (success_children, child_count, i) &&
-                    !afr_is_child_present (fresh_children, child_count, i)) {
-                        sh->child_errno[i] = ENOENT;
-                        GF_ASSERT (sh->xattr[i]);
-                        dict_unref (sh->xattr[i]);
-                        sh->xattr[i] = NULL;
-                }
-        }
-}
 
 int
-afr_sh_purge_stale_entries_done (call_frame_t *frame, xlator_t *this)
+afr_selfheal_inodelk (call_frame_t *frame, xlator_t *this, inode_t *inode,
+		      char *dom, off_t off, size_t size,
+		      unsigned char *locked_on)
 {
-        afr_local_t         *local    = NULL;
-        afr_self_heal_t     *sh       = NULL;
-        afr_private_t       *priv     = NULL;
-
-        local    = frame->local;
-        sh       = &local->self_heal;
-        priv     = this->private;
-
-        if (sh->op_failed) {
-                afr_sh_missing_entries_finish (frame, this);
-        } else {
-                if (afr_gfid_missing_count (this->name, sh->fresh_children,
-                                            sh->buf, priv->child_count,
-                                            local->loc.path)) {
-                        afr_sh_common_lookup (frame, this, &local->loc,
-                                              afr_sh_missing_entries_lookup_done,
-                                              sh->sh_gfid_req,
-                                              AFR_LOOKUP_FAIL_CONFLICTS|
-                                              AFR_LOOKUP_FAIL_MISSING_GFIDS,
-                                              NULL);
-                } else {
-                        //No need to set gfid so goto missing entries lookup done
-                        //Behave as if you have done the lookup
-                        afr_sh_remove_stale_lookup_info (sh,
-                                                         sh->success_children,
-                                                         sh->fresh_children,
-                                                         priv->child_count);
-                        afr_children_copy (sh->success_children,
-                                           sh->fresh_children,
-                                           priv->child_count);
-                        afr_sh_missing_entries_lookup_done (frame, this, 0, 0);
-                }
-        }
-        return 0;
-}
+	loc_t loc = {0,};
+	struct gf_flock flock = {0, };
+	afr_local_t *local = NULL;
+	int i = 0;
+	afr_private_t *priv = NULL;
 
-gf_boolean_t
-afr_sh_purge_entry_condition (afr_local_t *local, afr_private_t *priv,
-                              int child)
-{
-        afr_self_heal_t *sh = NULL;
+	priv = this->private;
+	local = frame->local;
 
-        sh = &local->self_heal;
+	loc.inode = inode_ref (inode);
+	uuid_copy (loc.gfid, inode->gfid);
 
-        if (local->child_up[child] &&
-            (!afr_is_child_present (sh->fresh_parent_dirs, priv->child_count,
-                                    child))
-            && (sh->child_errno[child] != ENOENT))
-                return _gf_true;
+	flock.l_type = F_WRLCK;
+	flock.l_start = off;
+	flock.l_len = size;
 
-        return _gf_false;
-}
+	AFR_ONALL (frame, afr_selfheal_lock_cbk, inodelk, dom,
+		    &loc, F_SETLK, &flock, NULL);
 
-gf_boolean_t
-afr_sh_purge_stale_entry_condition (afr_local_t *local, afr_private_t *priv,
-                                    int child)
-{
-        afr_self_heal_t *sh = NULL;
+	for (i = 0; i < priv->child_count; i++) {
+		if (local->replies[i].op_ret == -1 &&
+		    local->replies[i].op_errno == EAGAIN) {
+			afr_selfheal_locked_fill (frame, this, locked_on);
+			afr_selfheal_uninodelk (frame, this, inode, dom, off,
+						size, locked_on);
 
-        sh = &local->self_heal;
+			AFR_SEQ (frame, afr_selfheal_lock_cbk, inodelk, dom,
+				 &loc, F_SETLKW, &flock, NULL);
+			break;
+		}
+	}
 
-        if (local->child_up[child] &&
-            (!afr_is_child_present (sh->fresh_children, priv->child_count,
-                                    child))
-             && (sh->child_errno[child] != ENOENT))
-                return _gf_true;
+	loc_wipe (&loc);
 
-        return _gf_false;
+	return afr_selfheal_locked_fill (frame, this, locked_on);
 }
 
-void
-afr_sh_purge_entry_common (call_frame_t *frame, xlator_t *this,
-                           gf_boolean_t purge_condition (afr_local_t *local,
-                                                         afr_private_t *priv,
-                                                         int child))
-{
-        afr_local_t     *local = NULL;
-        afr_private_t   *priv = NULL;
-        afr_self_heal_t *sh = NULL;
-        int             i = 0;
-        int             call_count = 0;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (purge_condition (local, priv, i))
-                        call_count++;
-        }
-
-        if (call_count == 0) {
-                sh->post_remove_call (frame, this);
-                goto out;
-        }
-
-        local->call_count = call_count;
-        for (i = 0; i < priv->child_count; i++) {
-                if (!purge_condition (local, priv, i))
-                        continue;
-                gf_log (this->name, GF_LOG_INFO, "purging the stale entry %s "
-                        "on %d", local->loc.path, i);
-                afr_sh_call_entry_expunge_remove (frame, this,
-                                                  (long) i, &sh->buf[i],
-                                                  &sh->parentbufs[i],
-                                                  afr_sh_remove_entry_cbk);
-        }
-out:
-        return;
-}
 
-void
-afr_sh_purge_entry (call_frame_t *frame, xlator_t *this)
+int
+afr_selfheal_uninodelk (call_frame_t *frame, xlator_t *this, inode_t *inode,
+			char *dom, off_t off, size_t size,
+			const unsigned char *locked_on)
 {
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
+	loc_t loc = {0,};
+	struct gf_flock flock = {0, };
 
-        local = frame->local;
-        sh = &local->self_heal;
-        sh->post_remove_call = afr_sh_missing_entries_finish;
 
-        afr_sh_purge_entry_common (frame, this, afr_sh_purge_entry_condition);
-}
+	loc.inode = inode_ref (inode);
+	uuid_copy (loc.gfid, inode->gfid);
 
-void
-afr_sh_purge_stale_entry (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        int             i = 0;
+	flock.l_type = F_UNLCK;
+	flock.l_start = off;
+	flock.l_len = size;
 
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
+	AFR_ONLIST (locked_on, frame, afr_selfheal_lock_cbk, inodelk,
+		    dom, &loc, F_SETLK, &flock, NULL);
 
-        sh->post_remove_call = afr_sh_purge_stale_entries_done;
+	loc_wipe (&loc);
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (afr_is_child_present (sh->fresh_children,
-                                          priv->child_count, i))
-                        continue;
+	return 0;
+}
 
-                if ((!local->child_up[i]) || sh->child_errno[i] != 0)
-                        continue;
 
-                GF_ASSERT (!uuid_is_null (sh->entrybuf.ia_gfid) ||
-                           uuid_is_null (sh->buf[i].ia_gfid));
+int
+afr_selfheal_tryentrylk (call_frame_t *frame, xlator_t *this, inode_t *inode,
+			 char *dom, const char *name, unsigned char *locked_on)
+{
+	loc_t loc = {0,};
 
-                if ((sh->entrybuf.ia_type != sh->buf[i].ia_type) ||
-                    (uuid_compare (sh->buf[i].ia_gfid,
-                                   sh->entrybuf.ia_gfid)))
-                        continue;
+	loc.inode = inode_ref (inode);
+	uuid_copy (loc.gfid, inode->gfid);
 
-                afr_children_add_child (sh->fresh_children, i,
-                                        priv->child_count);
+	AFR_ONALL (frame, afr_selfheal_lock_cbk, entrylk, dom,
+		   &loc, name, ENTRYLK_LOCK_NB, ENTRYLK_WRLCK, NULL);
 
-        }
-        afr_sh_purge_entry_common (frame, this,
-                                   afr_sh_purge_stale_entry_condition);
-}
+	loc_wipe (&loc);
 
-void
-afr_sh_save_child_iatts_from_policy (int32_t *children, struct iatt *bufs,
-                                     struct iatt *save,
-                                     unsigned int child_count)
-{
-        int             i = 0;
-        int             child = 0;
-        gf_boolean_t    saved = _gf_false;
-
-        GF_ASSERT (save);
-        //if iatt buf with gfid exists sets it
-        for (i = 0; i < child_count; i++) {
-                child = children[i];
-                if (child == -1)
-                        break;
-                *save = bufs[child];
-                saved = _gf_true;
-                if (!uuid_is_null (save->ia_gfid))
-                        break;
-        }
-        GF_ASSERT (saved);
+	return afr_selfheal_locked_fill (frame, this, locked_on);
 }
 
-void
-afr_get_children_of_fresh_parent_dirs (afr_self_heal_t *sh,
-                                       unsigned int child_count)
-{
-        afr_children_intersection_get (sh->success_children,
-                                       sh->fresh_parent_dirs,
-                                       sh->sources, child_count);
-        afr_get_fresh_children (sh->success_children, sh->sources,
-                                sh->fresh_children, child_count);
-        memset (sh->sources, 0, sizeof (*sh->sources) * child_count);
-}
 
-void
-afr_sh_children_lookup_done (call_frame_t *frame, xlator_t *this,
-                             int32_t op_ret, int32_t op_errno)
+int
+afr_selfheal_entrylk (call_frame_t *frame, xlator_t *this, inode_t *inode,
+		      char *dom, const char *name, unsigned char *locked_on)
 {
-        afr_local_t      *local = NULL;
-        afr_self_heal_t  *sh = NULL;
-        afr_private_t    *priv = NULL;
-        int32_t          fresh_child_enoents = 0;
-        int32_t          fresh_parent_count = 0;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        if (op_ret < 0)
-                goto fail;
-        afr_get_children_of_fresh_parent_dirs (sh, priv->child_count);
-        fresh_parent_count = afr_get_children_count (sh->fresh_parent_dirs,
-                                                     priv->child_count);
-        //we need the enoent count of the subvols present in fresh_parent_dirs
-        fresh_child_enoents = afr_errno_count (sh->fresh_parent_dirs,
-                                               sh->child_errno,
-                                               priv->child_count, ENOENT);
-        if (fresh_child_enoents == fresh_parent_count) {
-                gf_log (this->name, GF_LOG_INFO, "Deleting stale file %s",
-                        local->loc.path);
-                afr_sh_set_error (sh, ENOENT);
-                sh->op_failed = 1;
-                afr_sh_purge_entry (frame, this);
-        } else if (!afr_conflicting_iattrs (sh->buf, sh->fresh_children,
-                                            priv->child_count, local->loc.path,
-                                            this->name)) {
-                afr_sh_save_child_iatts_from_policy (sh->fresh_children,
-                                                     sh->buf, &sh->entrybuf,
-                                                     priv->child_count);
-                afr_update_gfid_from_iatts (sh->sh_gfid_req, sh->buf,
-                                            sh->fresh_children,
-                                            priv->child_count);
-                afr_sh_purge_stale_entry (frame, this);
-        } else {
-                op_errno = EIO;
-                local->govinda_gOvinda = 1;
-                goto fail;
-        }
-
-        return;
-
-fail:
-        sh->op_failed = 1;
-        afr_sh_set_error (sh, op_errno);
-        afr_sh_missing_entries_finish (frame, this);
-        return;
-}
+	loc_t loc = {0,};
+	afr_local_t *local = NULL;
+	int i = 0;
+	afr_private_t *priv = NULL;
 
-static void
-afr_sh_find_fresh_parents (call_frame_t *frame, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno)
-{
-        afr_self_heal_t *sh  = NULL;
-        afr_private_t   *priv = NULL;
-        afr_local_t     *local = NULL;
-        int             enoent_count = 0;
-        int             nsources = 0;
-        int             source  = -1;
-        int32_t         subvol_status = 0;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        if (op_ret < 0)
-                goto out;
-        enoent_count = afr_errno_count (NULL, sh->child_errno,
-                                        priv->child_count, ENOENT);
-        if (enoent_count > 0) {
-                gf_log (this->name, GF_LOG_INFO, "Parent dir missing for %s,"
-                        " in missing entry self-heal, aborting missing-entry "
-                        "self-heal",
-                        local->loc.path);
-                afr_sh_missing_entries_finish (frame, this);
-                return;
-        }
-
-        nsources = afr_build_sources (this, sh->xattr, sh->buf,
-                                      sh->pending_matrix, sh->sources,
-                                      sh->success_children,
-                                      AFR_ENTRY_TRANSACTION, &subvol_status,
-                                      _gf_true);
-        if ((subvol_status & ALL_FOOLS) ||
-            (subvol_status & SPLIT_BRAIN)) {
-                gf_log (this->name, GF_LOG_INFO, "%s: Performing conservative "
-                        "merge", sh->parent_loc.path);
-                afr_mark_success_children_sources (sh->sources,
-                                                   sh->success_children,
-                                                   priv->child_count);
-        } else if (nsources < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "No sources for dir "
-                        "of %s, in missing entry self-heal, aborting "
-                        "self-heal", local->loc.path);
-                op_errno = EIO;
-                goto out;
-        }
-
-        source = afr_sh_select_source (sh->sources, priv->child_count);
-        if (source == -1) {
-                GF_ASSERT (0);
-                gf_log (this->name, GF_LOG_DEBUG, "No active sources found.");
-                op_errno = EIO;
-                goto out;
-        }
-        afr_get_fresh_children (sh->success_children, sh->sources,
-                                sh->fresh_parent_dirs, priv->child_count);
-        afr_sh_common_lookup (frame, this, &local->loc,
-                              afr_sh_children_lookup_done, NULL, 0,
-                              NULL);
-        return;
+	priv = this->private;
+	local = frame->local;
 
-out:
-        afr_sh_set_error (sh, op_errno);
-        sh->op_failed = 1;
-        afr_sh_missing_entries_finish (frame, this);
-        return;
-}
+	loc.inode = inode_ref (inode);
+	uuid_copy (loc.gfid, inode->gfid);
 
-void
-afr_sh_common_reset (afr_self_heal_t *sh, unsigned int child_count)
-{
-        int             i = 0;
-
-        for (i = 0; i < child_count; i++) {
-                memset (&sh->buf[i], 0, sizeof (sh->buf[i]));
-                memset (&sh->parentbufs[i], 0, sizeof (sh->parentbufs[i]));
-                sh->child_errno[i] = 0;
-        }
-        memset (&sh->parentbuf, 0, sizeof (sh->parentbuf));
-        sh->success_count = 0;
-        afr_reset_children (sh->success_children, child_count);
-        afr_reset_children (sh->fresh_children, child_count);
-        afr_reset_xattr (sh->xattr, child_count);
-        loc_wipe (&sh->lookup_loc);
-}
+	AFR_ONALL (frame, afr_selfheal_lock_cbk, entrylk, dom, &loc,
+		   name, ENTRYLK_LOCK_NB, ENTRYLK_WRLCK, NULL);
 
-/* afr self-heal state will be lost if this call is made
- * please check the afr_sh_common_reset that is called in this function
- */
-int
-afr_sh_common_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                      afr_lookup_done_cbk_t lookup_done , uuid_t gfid,
-                      int32_t flags, dict_t *xdata)
-{
-        afr_local_t    *local = NULL;
-        int             i = 0;
-        int             call_count = 0;
-        afr_private_t  *priv = NULL;
-        dict_t         *xattr_req = NULL;
-        afr_self_heal_t *sh = NULL;
-
-        local = frame->local;
-        priv  = this->private;
-        sh    = &local->self_heal;
-
-        call_count = afr_up_children_count (local->child_up, priv->child_count);
-
-        local->call_count = call_count;
-
-        xattr_req = dict_new();
-
-        if (xattr_req) {
-                afr_xattr_req_prepare (this, xattr_req, loc->path);
-                if (gfid) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "looking up %s with gfid: %s",
-                                loc->path, uuid_utoa (gfid));
-                        GF_ASSERT (!uuid_is_null (gfid));
-                        afr_set_dict_gfid (xattr_req, gfid);
-                }
-        }
-
-        afr_sh_common_reset (sh, priv->child_count);
-        sh->lookup_done = lookup_done;
-        loc_copy (&sh->lookup_loc, loc);
-        sh->lookup_flags = flags;
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "looking up %s on subvolume %s",
-                                loc->path, priv->children[i]->name);
-
-                        STACK_WIND_COOKIE (frame,
-                                           afr_sh_common_lookup_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->lookup,
-                                           loc, xattr_req);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
-
-        if (xattr_req)
-                dict_unref (xattr_req);
-
-        return 0;
-}
+	for (i = 0; i < priv->child_count; i++) {
+		if (local->replies[i].op_ret == -1 &&
+		    local->replies[i].op_errno == EAGAIN) {
+			afr_selfheal_locked_fill (frame, this, locked_on);
+			afr_selfheal_unentrylk (frame, this, inode, dom, name,
+						locked_on);
 
+			AFR_SEQ (frame, afr_selfheal_lock_cbk, entrylk, dom,
+				 &loc, name, ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL);
+			break;
+		}
+	}
 
+	loc_wipe (&loc);
 
-int
-afr_sh_post_nb_entrylk_conflicting_sh_cbk (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_self_heal_t     *sh       = NULL;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        sh       = &local->self_heal;
-
-        if (int_lock->lock_op_ret < 0) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "Non blocking entrylks failed.");
-                sh->op_failed = -1;
-                afr_sh_missing_entries_done (frame, this);
-        } else {
-
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Non blocking entrylks done. Proceeding to FOP");
-                afr_sh_common_lookup (frame, this, &sh->parent_loc,
-                                      afr_sh_find_fresh_parents,
-                                      NULL, AFR_LOOKUP_FAIL_CONFLICTS,
-                                      NULL);
-        }
-
-        return 0;
+	return afr_selfheal_locked_fill (frame, this, locked_on);
 }
 
+
 int
-afr_sh_post_nb_entrylk_gfid_sh_cbk (call_frame_t *frame, xlator_t *this)
+afr_selfheal_unentrylk (call_frame_t *frame, xlator_t *this, inode_t *inode,
+			char *dom, const char *name, unsigned char *locked_on)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_self_heal_t     *sh       = NULL;
-
-        local    = frame->local;
-        sh       = &local->self_heal;
-        int_lock = &local->internal_lock;
-
-        if (int_lock->lock_op_ret < 0) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "Non blocking entrylks failed.");
-                afr_sh_missing_entries_done (frame, this);
-        } else {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Non blocking entrylks done. Proceeding to FOP");
-                afr_sh_common_lookup (frame, this, &local->loc,
-                                      afr_sh_missing_entries_lookup_done,
-                                      sh->sh_gfid_req, AFR_LOOKUP_FAIL_CONFLICTS|
-                                      AFR_LOOKUP_FAIL_MISSING_GFIDS,
-                                      NULL);
-        }
-
-        return 0;
+	loc_t loc = {0,};
+
+	loc.inode = inode_ref (inode);
+	uuid_copy (loc.gfid, inode->gfid);
+
+	AFR_ONLIST (locked_on, frame, afr_selfheal_lock_cbk, entrylk,
+		    dom, &loc, name, ENTRYLK_UNLOCK, ENTRYLK_WRLCK, NULL);
+
+	loc_wipe (&loc);
+
+	return 0;
 }
 
-int
-afr_sh_entrylk (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                char *base_name, afr_lock_cbk_t lock_cbk)
+
+gf_boolean_t
+afr_is_pending_set (xlator_t *this, dict_t *xdata, int type)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
+	int idx = -1;
+	afr_private_t *priv = NULL;
+	void *pending_raw = NULL;
+	int *pending_int = NULL;
+	int i = 0;
 
-        local    = frame->local;
-        int_lock = &local->internal_lock;
+	priv = this->private;
+	idx = afr_index_for_transaction_type (type);
 
-        int_lock->transaction_lk_type = AFR_SELFHEAL_LK;
-        int_lock->selfheal_lk_type    = AFR_ENTRY_SELF_HEAL_LK;
+	if (dict_get_ptr (xdata, AFR_DIRTY, &pending_raw) == 0) {
+		if (pending_raw) {
+			pending_int = pending_raw;
 
-        afr_set_lock_number (frame, this);
+			if (ntoh32 (pending_int[idx]))
+				return _gf_true;
+		}
+	}
 
-        int_lock->lk_basename = base_name;
-        int_lock->lk_loc      = loc;
-        int_lock->lock_cbk    = lock_cbk;
+	for (i = 0; i < priv->child_count; i++) {
+		if (dict_get_ptr (xdata, priv->pending_key[i],
+				  &pending_raw))
+			continue;
+		if (!pending_raw)
+			continue;
+		pending_int = pending_raw;
 
-        afr_nonblocking_entrylk (frame, this);
+		if (ntoh32 (pending_int[idx]))
+			return _gf_true;
+	}
 
-        return 0;
+	return _gf_false;
 }
 
-static int
-afr_self_heal_parent_entrylk (call_frame_t *frame, xlator_t *this,
-                              afr_lock_cbk_t lock_cbk)
+
+gf_boolean_t
+afr_is_data_set (xlator_t *this, dict_t *xdata)
 {
-        afr_local_t         *local    = NULL;
-        afr_self_heal_t     *sh       = NULL;
-        afr_internal_lock_t *int_lock = NULL;
-        int                 ret       = -1;
-        int32_t             op_errno  = 0;
-
-        local    = frame->local;
-        sh       = &local->self_heal;
-
-        gf_log (this->name, GF_LOG_TRACE,
-                "attempting to recreate missing entries for path=%s",
-                local->loc.path);
-
-        ret = afr_build_parent_loc (&sh->parent_loc, &local->loc, &op_errno);
-        if (ret)
-                goto out;
-
-        afr_sh_entrylk (frame, this, &sh->parent_loc, NULL,
-                        lock_cbk);
-        return 0;
-out:
-        int_lock = &local->internal_lock;
-        int_lock->lock_op_ret = -1;
-        lock_cbk (frame, this);
-        return 0;
+	return afr_is_pending_set (this, xdata, AFR_DATA_TRANSACTION);
 }
 
-static int
-afr_self_heal_conflicting_entries (call_frame_t *frame, xlator_t *this)
+gf_boolean_t
+afr_is_metadata_set (xlator_t *this, dict_t *xdata)
 {
-        afr_self_heal_parent_entrylk (frame, this,
-                                      afr_sh_post_nb_entrylk_conflicting_sh_cbk);
-        return 0;
+	return afr_is_pending_set (this, xdata, AFR_METADATA_TRANSACTION);
 }
 
-static int
-afr_self_heal_gfids (call_frame_t *frame, xlator_t *this)
+gf_boolean_t
+afr_is_entry_set (xlator_t *this, dict_t *xdata)
 {
-        afr_self_heal_parent_entrylk (frame, this,
-                                      afr_sh_post_nb_entrylk_gfid_sh_cbk);
-        return 0;
+	return afr_is_pending_set (this, xdata, AFR_ENTRY_TRANSACTION);
 }
 
-afr_local_t *afr_local_copy (afr_local_t *l, xlator_t *this)
+
+void
+afr_inode_link (inode_t *inode, struct iatt *iatt)
 {
-        afr_private_t *priv = NULL;
-        afr_local_t   *lc     = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_self_heal_t *shc = NULL;
-
-        priv = this->private;
-
-        sh = &l->self_heal;
-
-        lc = mem_get0 (this->local_pool);
-        if (!lc)
-                goto out;
-
-        shc = &lc->self_heal;
-
-        shc->unwind = sh->unwind;
-        shc->gfid_sh_success_cbk = sh->gfid_sh_success_cbk;
-        shc->do_missing_entry_self_heal = sh->do_missing_entry_self_heal;
-        shc->do_gfid_self_heal = sh->do_gfid_self_heal;
-        shc->do_data_self_heal = sh->do_data_self_heal;
-        shc->do_metadata_self_heal = sh->do_metadata_self_heal;
-        shc->do_entry_self_heal = sh->do_entry_self_heal;
-        shc->forced_merge = sh->forced_merge;
-        shc->background = sh->background;
-        shc->type = sh->type;
-
-        uuid_copy (shc->sh_gfid_req, sh->sh_gfid_req);
-        if (l->loc.path)
-                loc_copy (&lc->loc, &l->loc);
-
-        lc->child_up  = memdup (l->child_up,
-                                sizeof (*lc->child_up) * priv->child_count);
-        if (l->xattr_req)
-                lc->xattr_req = dict_ref (l->xattr_req);
-
-        if (l->cont.lookup.inode)
-                lc->cont.lookup.inode = inode_ref (l->cont.lookup.inode);
-        if (l->cont.lookup.xattr)
-                lc->cont.lookup.xattr = dict_ref (l->cont.lookup.xattr);
-        if (l->internal_lock.inode_locked_nodes)
-                lc->internal_lock.inode_locked_nodes =
-                        memdup (l->internal_lock.inode_locked_nodes,
-                                sizeof (*lc->internal_lock.inode_locked_nodes) * priv->child_count);
-        else
-                lc->internal_lock.inode_locked_nodes =
-                        GF_CALLOC (sizeof (*l->internal_lock.inode_locked_nodes),
-                                   priv->child_count,
-                                   gf_afr_mt_char);
-        if (l->internal_lock.entry_locked_nodes)
-                lc->internal_lock.entry_locked_nodes =
-                        memdup (l->internal_lock.entry_locked_nodes,
-                                sizeof (*lc->internal_lock.entry_locked_nodes) * priv->child_count);
-        else
-                lc->internal_lock.entry_locked_nodes =
-                        GF_CALLOC (sizeof (*l->internal_lock.entry_locked_nodes),
-                                   priv->child_count,
-                                   gf_afr_mt_char);
-        if (l->internal_lock.locked_nodes)
-                lc->internal_lock.locked_nodes =
-                        memdup (l->internal_lock.locked_nodes,
-                                sizeof (*lc->internal_lock.locked_nodes) * priv->child_count);
-        else
-                lc->internal_lock.locked_nodes =
-                        GF_CALLOC (sizeof (*l->internal_lock.locked_nodes),
-                                   priv->child_count,
-                                   gf_afr_mt_char);
-
-        lc->internal_lock.inodelk_lock_count =
-                l->internal_lock.inodelk_lock_count;
-        lc->internal_lock.entrylk_lock_count =
-                l->internal_lock.entrylk_lock_count;
+	inode_t *linked_inode = NULL;
 
-out:
-        return lc;
+	linked_inode = inode_link (inode, NULL, NULL, iatt);
+
+	uuid_copy (inode->gfid, iatt->ia_gfid);
+	inode->ia_type = iatt->ia_type;
+
+	if (linked_inode) {
+		inode_lookup (linked_inode);
+		inode_unref (linked_inode);
+	}
 }
 
+
+/*
+ * This function inspects the looked up replies (in an unlocked manner)
+ * and decides whether a locked verification and possible healing is
+ * required or not. It updates the three booleans for each type
+ * of healing. If the boolean flag gets set to FALSE, then we are sure
+ * no healing is required. If the boolean flag gets set to TRUE then
+ * we have to proceed with locked reinspection.
+ */
+
 int
-afr_self_heal_completion_cbk (call_frame_t *bgsh_frame, xlator_t *this)
-{
-        afr_private_t *   priv  = NULL;
-        afr_local_t *     local = NULL;
-        afr_self_heal_t * sh    = NULL;
-        char              sh_type_str[256] = {0,};
-        gf_boolean_t      split_brain = _gf_false;
+afr_selfheal_unlocked_inspect (call_frame_t *frame, xlator_t *this,
+			       inode_t *inode, uuid_t gfid,
+			       gf_boolean_t *data_selfheal,
+			       gf_boolean_t *metadata_selfheal,
+			       gf_boolean_t *entry_selfheal)
+{
+	afr_private_t *priv = NULL;
+	int i = 0;
+	int valid_cnt = 0;
+	struct iatt first = {0, };
+	struct afr_reply *replies = NULL;
+	int ret = -1;
+
+	priv = this->private;
+
+	replies = alloca0 (sizeof (*replies) * priv->child_count);
+
+	ret = afr_selfheal_unlocked_discover (frame, inode, gfid, replies);
+	if (ret)
+		return ret;
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!replies[i].valid)
+			continue;
+		if (replies[i].op_ret == -1)
+			continue;
+
+		if (afr_is_data_set (this, replies[i].xdata))
+			*data_selfheal = _gf_true;
+
+		if (afr_is_metadata_set (this, replies[i].xdata))
+			*metadata_selfheal = _gf_true;
 
-        priv  = this->private;
-        local = bgsh_frame->local;
-        sh    = &local->self_heal;
+		if (afr_is_entry_set (this, replies[i].xdata))
+			*entry_selfheal = _gf_true;
 
-        if (local->govinda_gOvinda || sh->mdata_spb || sh->data_spb)
-                split_brain = _gf_true;
+		valid_cnt ++;
+		if (valid_cnt == 1) {
+			first = replies[i].poststat;
+			continue;
+		}
 
-        afr_set_split_brain (this, sh->inode, split_brain);
+		if (!IA_EQUAL (first, replies[i].poststat, type)) {
+			gf_log (this->name, GF_LOG_ERROR,
+				"TYPE mismatch %d vs %d on %s for gfid:%s",
+				(int) first.ia_type,
+				(int) replies[i].poststat.ia_type,
+				priv->children[i]->name,
+				uuid_utoa (replies[i].poststat.ia_gfid));
+			return -EIO;
+		}
 
-        afr_self_heal_type_str_get (sh, sh_type_str,
-                                    sizeof(sh_type_str));
-        if (sh->op_failed) {
-                gf_loglevel_t loglevel = GF_LOG_ERROR;
-                if (priv->shd.iamshd)
-                        loglevel = GF_LOG_DEBUG;
+		if (!IA_EQUAL (first, replies[i].poststat, uid)) {
+			gf_log (this->name, GF_LOG_DEBUG,
+				"UID mismatch %d vs %d on %s for gfid:%s",
+				(int) first.ia_uid,
+				(int) replies[i].poststat.ia_uid,
+				priv->children[i]->name,
+				uuid_utoa (replies[i].poststat.ia_gfid));
 
-                gf_log (this->name, loglevel, "background %s self-heal "
-                        "failed on %s", sh_type_str, local->loc.path);
+			*metadata_selfheal = _gf_true;
+		}
 
-        } else {
-                gf_log (this->name, GF_LOG_DEBUG, "background %s self-heal "
-                        "completed on %s", sh_type_str, local->loc.path);
+		if (!IA_EQUAL (first, replies[i].poststat, gid)) {
+			gf_log (this->name, GF_LOG_DEBUG,
+				"GID mismatch %d vs %d on %s for gfid:%s",
+				(int) first.ia_uid,
+				(int) replies[i].poststat.ia_uid,
+				priv->children[i]->name,
+				uuid_utoa (replies[i].poststat.ia_gfid));
 
-        }
+			*metadata_selfheal = _gf_true;
+		}
 
-        FRAME_SU_UNDO (bgsh_frame, afr_local_t);
+		if (!IA_EQUAL (first, replies[i].poststat, prot)) {
+			gf_log (this->name, GF_LOG_DEBUG,
+				"MODE mismatch %d vs %d on %s for gfid:%s",
+				(int) st_mode_from_ia (first.ia_prot, 0),
+				(int) st_mode_from_ia (replies[i].poststat.ia_prot, 0),
+				priv->children[i]->name,
+				uuid_utoa (replies[i].poststat.ia_gfid));
 
-        if (!sh->unwound && sh->unwind) {
-                sh->unwind (sh->orig_frame, this, sh->op_ret, sh->op_errno,
-                            sh->op_failed);
-        }
+			*metadata_selfheal = _gf_true;
+		}
 
-        if (sh->background) {
-                LOCK (&priv->lock);
-                {
-                        priv->background_self_heals_started--;
-                }
-                UNLOCK (&priv->lock);
-        }
+		if (IA_ISREG(first.ia_type) &&
+		    !IA_EQUAL (first, replies[i].poststat, size)) {
+			gf_log (this->name, GF_LOG_DEBUG,
+				"SIZE mismatch %lld vs %lld on %s for gfid:%s",
+				(long long) first.ia_size,
+				(long long) replies[i].poststat.ia_size,
+				priv->children[i]->name,
+				uuid_utoa (replies[i].poststat.ia_gfid));
 
-        AFR_STACK_DESTROY (bgsh_frame);
+			*data_selfheal = _gf_true;
+		}
+	}
 
-        return 0;
-}
+	if (valid_cnt > 0)
+		afr_inode_link (inode, &first);
 
-int
-afr_self_heal (call_frame_t *frame, xlator_t *this, inode_t *inode)
-{
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        int32_t          op_errno = 0;
-        int              ret = 0;
-        afr_self_heal_t *orig_sh = NULL;
-        call_frame_t    *sh_frame = NULL;
-        afr_local_t     *sh_local = NULL;
-        loc_t           *loc   = NULL;
-
-        local = frame->local;
-        orig_sh = &local->self_heal;
-        priv  = this->private;
-
-        GF_ASSERT (local->loc.path);
-
-        gf_log (this->name, GF_LOG_TRACE,
-                "performing self heal on %s (metadata=%d data=%d entry=%d)",
-                local->loc.path,
-                local->self_heal.do_metadata_self_heal,
-                local->self_heal.do_data_self_heal,
-                local->self_heal.do_entry_self_heal);
-
-        op_errno        = ENOMEM;
-        sh_frame        = copy_frame (frame);
-        if (!sh_frame)
-                goto out;
-        afr_set_lk_owner (sh_frame, this, sh_frame->root);
-        afr_set_low_priority (sh_frame);
-
-        sh_local        = afr_local_copy (local, this);
-        if (!sh_local)
-                goto out;
-        sh_frame->local = sh_local;
-        sh              = &sh_local->self_heal;
-
-        sh->inode       = inode_ref (inode);
-        sh->orig_frame  = frame;
-
-        sh->completion_cbk = afr_self_heal_completion_cbk;
-
-        sh->success = GF_CALLOC (priv->child_count, sizeof (*sh->success),
-                                 gf_afr_mt_char);
-        if (!sh->success)
-                goto out;
-        sh->sources = GF_CALLOC (sizeof (*sh->sources), priv->child_count,
-                                 gf_afr_mt_int);
-        if (!sh->sources)
-                goto out;
-        sh->locked_nodes = GF_CALLOC (sizeof (*sh->locked_nodes),
-                                      priv->child_count,
-                                      gf_afr_mt_int);
-        if (!sh->locked_nodes)
-                goto out;
-
-        sh->pending_matrix = afr_matrix_create (priv->child_count,
-                                                priv->child_count);
-        if (!sh->pending_matrix)
-                goto out;
-
-        sh->delta_matrix = afr_matrix_create (priv->child_count,
-                                              priv->child_count);
-        if (!sh->delta_matrix)
-                goto out;
-
-        sh->fresh_parent_dirs = afr_children_create (priv->child_count);
-        if (!sh->fresh_parent_dirs)
-                goto out;
-        ret = afr_sh_common_create (sh, priv->child_count);
-        if (ret) {
-                op_errno = -ret;
-                goto out;
-        }
-
-        if (local->self_heal.background) {
-                LOCK (&priv->lock);
-                {
-                        if (priv->background_self_heals_started
-                            < priv->background_self_heal_count) {
-                                priv->background_self_heals_started++;
-
-
-                        } else {
-                                local->self_heal.background = _gf_false;
-                                sh->background = _gf_false;
-                        }
-                }
-                UNLOCK (&priv->lock);
-        }
-
-        if (!local->loc.parent) {
-                sh->do_missing_entry_self_heal = _gf_false;
-                sh->do_gfid_self_heal = _gf_false;
-        }
-
-        FRAME_SU_DO (sh_frame, afr_local_t);
-        if (sh->do_missing_entry_self_heal) {
-                afr_self_heal_conflicting_entries (sh_frame, this);
-        } else if (sh->do_gfid_self_heal) {
-                GF_ASSERT (!uuid_is_null (sh->sh_gfid_req));
-                afr_self_heal_gfids (sh_frame, this);
-        } else {
-                loc = &sh_local->loc;
-                if (uuid_is_null (loc->inode->gfid) && uuid_is_null (loc->gfid)) {
-                        if (!uuid_is_null (inode->gfid))
-                                GF_ASSERT (!uuid_compare (inode->gfid,
-                                           sh->sh_gfid_req));
-                        uuid_copy (loc->gfid, sh->sh_gfid_req);
-                }
-                gf_log (this->name, GF_LOG_TRACE,
-                        "proceeding to metadata check on %s",
-                        local->loc.path);
-
-                afr_sh_missing_entries_done (sh_frame, this);
-        }
-        op_errno = 0;
+	if (valid_cnt < 2)
+		return -ENOTCONN;
 
-out:
-        if (op_errno) {
-                orig_sh->unwind (frame, this, -1, op_errno, 1);
-                if (sh_frame)
-                        AFR_STACK_DESTROY (sh_frame);
-        }
-        return 0;
+	return 0;
 }
 
-void
-afr_self_heal_type_str_get (afr_self_heal_t *self_heal_p, char *str,
-                            size_t size)
+
+inode_t *
+afr_inode_find (xlator_t *this, uuid_t gfid)
 {
-        GF_ASSERT (str && (size > strlen (" missing-entry gfid "
-                                          "meta-data data entry")));
+	inode_table_t *table = NULL;
+	inode_t *inode = NULL;
 
-        if (self_heal_p->do_metadata_self_heal) {
-                snprintf (str, size, " meta-data");
-        }
+	table = this->itable;
+	if (!table)
+		return NULL;
 
-        if (self_heal_p->do_data_self_heal) {
-                snprintf (str + strlen(str), size - strlen(str), " data");
-        }
+	inode = inode_find (table, gfid);
+	if (inode)
+		return inode;
 
-        if (self_heal_p->do_entry_self_heal) {
-                snprintf (str + strlen(str), size - strlen(str), " entry");
-        }
+	inode = inode_new (table);
+	if (!inode)
+		return NULL;
 
-        if (self_heal_p->do_missing_entry_self_heal) {
-                snprintf (str + strlen(str), size - strlen(str),
-                         " missing-entry");
-        }
+	uuid_copy (inode->gfid, gfid);
 
-        if (self_heal_p->do_gfid_self_heal) {
-                snprintf (str + strlen(str), size - strlen(str), " gfid");
-        }
+	return inode;
 }
 
-afr_self_heal_type
-afr_self_heal_type_for_transaction (afr_transaction_type type)
+
+call_frame_t *
+afr_frame_create (xlator_t *this)
 {
-        afr_self_heal_type sh_type = AFR_SELF_HEAL_INVALID;
-
-        switch (type) {
-        case AFR_DATA_TRANSACTION:
-                sh_type = AFR_SELF_HEAL_DATA;
-                break;
-        case AFR_METADATA_TRANSACTION:
-                sh_type = AFR_SELF_HEAL_METADATA;
-                break;
-        case AFR_ENTRY_TRANSACTION:
-                sh_type = AFR_SELF_HEAL_ENTRY;
-                break;
-        case AFR_ENTRY_RENAME_TRANSACTION:
-                GF_ASSERT (0);
-                break;
-        }
-        return sh_type;
+	call_frame_t *frame = NULL;
+	afr_local_t *local = NULL;
+	int op_errno = 0;
+	pid_t pid = -1;
+
+	frame = create_frame (this, this->ctx->pool);
+	if (!frame)
+		return NULL;
+
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local) {
+		STACK_DESTROY (frame->root);
+		return NULL;
+	}
+
+	syncopctx_setfspid (&pid);
+
+	frame->root->pid = pid;
+
+	afr_set_lk_owner (frame, this, frame->root);
+
+	return frame;
 }
 
-int
-afr_build_child_loc (xlator_t *this, loc_t *child, loc_t *parent, char *name)
-{
-        int   ret = -1;
-        uuid_t pargfid = {0};
 
-        if (!child)
-                goto out;
+/*
+ * This is the entry point for healing a given GFID
+ */
 
-        if (!uuid_is_null (parent->inode->gfid))
-                uuid_copy (pargfid, parent->inode->gfid);
-        else if (!uuid_is_null (parent->gfid))
-                uuid_copy (pargfid, parent->gfid);
+int
+afr_selfheal (xlator_t *this, uuid_t gfid)
+{
+	inode_t *inode = NULL;
+	call_frame_t *frame = NULL;
+	int ret = -1;
+	gf_boolean_t data_selfheal = _gf_false;
+	gf_boolean_t metadata_selfheal = _gf_false;
+	gf_boolean_t entry_selfheal = _gf_false;
 
-        if (uuid_is_null (pargfid))
-                goto out;
+	inode = afr_inode_find (this, gfid);
+	if (!inode)
+		goto out;
 
-        if (strcmp (parent->path, "/") == 0)
-                ret = gf_asprintf ((char **)&child->path, "/%s", name);
-        else
-                ret = gf_asprintf ((char **)&child->path, "%s/%s", parent->path,
-                                   name);
+	frame = afr_frame_create (this);
+	if (!frame)
+		goto out;
 
-        if (-1 == ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "asprintf failed while setting child path");
-        }
+	ret = afr_selfheal_unlocked_inspect (frame, this, inode, gfid,
+					     &data_selfheal,
+					     &metadata_selfheal,
+					     &entry_selfheal);
+	if (ret)
+		goto out;
 
-        child->name = strrchr (child->path, '/');
-        if (child->name)
-                child->name++;
+	if (data_selfheal)
+		afr_selfheal_data (frame, this, inode);
 
-        child->parent = inode_ref (parent->inode);
-        child->inode = inode_new (parent->inode->table);
-        uuid_copy (child->pargfid, pargfid);
+	if (metadata_selfheal)
+		afr_selfheal_metadata (frame, this, inode);
 
-        if (!child->inode) {
-                ret = -1;
-                goto out;
-        }
+	if (entry_selfheal)
+		afr_selfheal_entry (frame, this, inode);
 
-        ret = 0;
+	inode_forget (inode, 1);
 out:
-        if ((ret == -1) && child)
-                loc_wipe (child);
+	if (inode)
+		inode_unref (inode);
+	if (frame)
+		AFR_STACK_DESTROY (frame);
 
-        return ret;
+	return ret;
 }
diff --git a/xlators/cluster/afr/src/afr-self-heal-common.h b/xlators/cluster/afr/src/afr-self-heal-common.h
deleted file mode 100644
index d104593ac..000000000
--- a/xlators/cluster/afr/src/afr-self-heal-common.h
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#ifndef __AFR_SELF_HEAL_COMMON_H__
-#define __AFR_SELF_HEAL_COMMON_H__
-
-#define FILE_HAS_HOLES(buf) (((buf)->ia_size) > ((buf)->ia_blocks * 512))
-#define AFR_SH_MIN_PARTICIPANTS 2
-
-typedef enum {
-        AFR_SELF_HEAL_ENTRY,
-        AFR_SELF_HEAL_METADATA,
-        AFR_SELF_HEAL_DATA,
-        AFR_SELF_HEAL_INVALID = -1,
-} afr_self_heal_type;
-
-typedef enum {
-        AFR_LOOKUP_FAIL_CONFLICTS = 1,
-        AFR_LOOKUP_FAIL_MISSING_GFIDS = 2,
-} afr_lookup_flags_t;
-
-int
-afr_sh_select_source (int sources[], int child_count);
-
-int
-afr_sh_source_count (int sources[], int child_count);
-
-void
-afr_sh_print_pending_matrix (int32_t *pending_matrix[], xlator_t *this);
-
-int
-afr_build_pending_matrix (char **pending_key, int32_t **pending_matrix,
-                          unsigned char *ignorant_subvols,
-                          dict_t *xattr[], afr_transaction_type type,
-                          size_t child_count);
-
-void
-afr_sh_pending_to_delta (afr_private_t *priv, dict_t **xattr,
-                         int32_t *delta_matrix[], unsigned char success[],
-                         int child_count, afr_transaction_type type);
-
-int
-afr_mark_sources (xlator_t *this, int32_t *sources, int32_t **pending_matrix,
-                  struct iatt *bufs, afr_self_heal_type type,
-                  int32_t *success_children, int32_t *subvol_status);
-
-int
-afr_sh_delta_to_xattr (afr_private_t *priv,
-                       int32_t *delta_matrix[], dict_t *xattr[],
-		       int child_count, afr_transaction_type type);
-
-void
-afr_self_heal_type_str_get (afr_self_heal_t *self_heal_p, char *str,
-                            size_t size);
-
-afr_self_heal_type
-afr_self_heal_type_for_transaction (afr_transaction_type type);
-
-int
-afr_build_sources (xlator_t *this, dict_t **xattr, struct iatt *bufs,
-                   int32_t **pending_matrix, int32_t *sources,
-                   int32_t *success_children, afr_transaction_type type,
-                   int32_t *subvol_status, gf_boolean_t ignore_ignorant);
-void
-afr_sh_common_reset (afr_self_heal_t *sh, unsigned int child_count);
-
-void
-afr_sh_common_lookup_resp_handler (call_frame_t *frame, void *cookie,
-                                   xlator_t *this,
-                                   int32_t op_ret, int32_t op_errno,
-                                   inode_t *inode, struct iatt *buf,
-                                   dict_t *xattr, struct iatt *postparent,
-                                   loc_t *loc);
-
-int
-afr_sh_common_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                      afr_lookup_done_cbk_t lookup_cbk, uuid_t uuid,
-                      int32_t flags, dict_t *xdata);
-int
-afr_sh_entry_expunge_remove (call_frame_t *expunge_frame, xlator_t *this,
-                             int active_src, struct iatt *buf,
-                             struct iatt *parentbuf);
-int
-afr_sh_entrylk (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                char *base_name, afr_lock_cbk_t lock_cbk);
-int
-afr_sh_entry_impunge_create (call_frame_t *impunge_frame, xlator_t *this,
-                             int child_index);
-int
-afr_sh_data_unlock (call_frame_t *frame, xlator_t *this,
-                    afr_lock_cbk_t lock_cbk);
-afr_local_t *
-afr_local_copy (afr_local_t *l, xlator_t *this);
-int
-afr_sh_data_lock (call_frame_t *frame, xlator_t *this,
-                  off_t start, off_t len,
-                  afr_lock_cbk_t success_handler,
-                  afr_lock_cbk_t failure_handler);
-void
-afr_sh_set_error (afr_self_heal_t *sh, int32_t op_errno);
-void
-afr_sh_mark_source_sinks (call_frame_t *frame, xlator_t *this);
-typedef int
-(*afr_fxattrop_cbk_t) (call_frame_t *frame, void *cookie,
-                       xlator_t *this, int32_t op_ret, int32_t op_errno,
-                       dict_t *xattr, dict_t *xdata);
-int
-afr_build_child_loc (xlator_t *this, loc_t *child, loc_t *parent, char *name);
-int
-afr_impunge_frame_create (call_frame_t *frame, xlator_t *this,
-                          int active_source, call_frame_t **impunge_frame);
-void
-afr_sh_reset (call_frame_t *frame, xlator_t *this);
-
-void
-afr_children_intersection_get (int32_t *set1, int32_t *set2,
-                               int *intersection, unsigned int child_count);
-int
-afr_get_no_xattr_dir_read_child (xlator_t *this, int32_t *success_children,
-                                 struct iatt *bufs);
-#endif /* __AFR_SELF_HEAL_COMMON_H__ */
diff --git a/xlators/cluster/afr/src/afr-self-heal-data.c b/xlators/cluster/afr/src/afr-self-heal-data.c
index 70395be86..c0548d995 100644
--- a/xlators/cluster/afr/src/afr-self-heal-data.c
+++ b/xlators/cluster/afr/src/afr-self-heal-data.c
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -8,1414 +8,628 @@
   cases as published by the Free Software Foundation.
 */
 
-#include <libgen.h>
-#include <unistd.h>
-#include <fnmatch.h>
-#include <sys/time.h>
-#include <stdlib.h>
-#include <signal.h>
 
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
 #endif
 
-#include "glusterfs.h"
 #include "afr.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "stack.h"
-#include "list.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include "compat-errno.h"
-#include "compat.h"
-#include "byte-order.h"
-
-#include "afr-transaction.h"
 #include "afr-self-heal.h"
-#include "afr-self-heal-common.h"
-#include "afr-self-heal-algorithm.h"
-
-int
-afr_sh_data_fail (call_frame_t *frame, xlator_t *this);
-
-static inline gf_boolean_t
-afr_sh_data_proceed (unsigned int success_count)
-{
-        return (success_count >= AFR_SH_MIN_PARTICIPANTS);
-}
-
-extern int
-sh_loop_finish (call_frame_t *loop_frame, xlator_t *this);
-
-int
-afr_post_sh_big_lock_success (call_frame_t *frame, xlator_t *this);
-
-int
-afr_post_sh_big_lock_failure (call_frame_t *frame, xlator_t *this);
-
-int
-afr_sh_data_finish (call_frame_t *frame, xlator_t *this);
-
-int
-afr_sh_data_done (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-
-        local = frame->local;
-        sh = &local->self_heal;
-
-        sh->completion_cbk (frame, this);
+#include "byte-order.h"
 
-        return 0;
-}
+enum {
+	AFR_SELFHEAL_DATA_FULL = 0,
+	AFR_SELFHEAL_DATA_DIFF,
+};
 
 
-int
-afr_sh_data_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
+#define HAS_HOLES(i) ((i->ia_blocks * 512) < (i->ia_size))
+static int
+__checksum_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		int op_ret, int op_errno, uint32_t weak, uint8_t *strong,
+		dict_t *xdata)
 {
-        afr_local_t   *local       = NULL;
-        afr_private_t *priv        = NULL;
-        int            call_count  = 0;
-        int            child_index = (long) cookie;
-
-        local = frame->local;
-        priv = this->private;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "flush failed on %s on subvolume %s: %s",
-                                local->loc.path, priv->children[child_index]->name,
-                                strerror (op_errno));
-                }
-        }
-        UNLOCK (&frame->lock);
+	afr_local_t *local = NULL;
+	int i = (long) cookie;
 
-        call_count = afr_frame_return (frame);
+	local = frame->local;
 
-        if (call_count == 0) {
-                afr_sh_data_done (frame, this);
-        }
+	local->replies[i].valid = 1;
+	local->replies[i].op_ret = op_ret;
+	local->replies[i].op_errno = op_errno;
+	if (strong)
+		memcpy (local->replies[i].checksum, strong, MD5_DIGEST_LENGTH);
 
-        return 0;
+	syncbarrier_wake (&local->barrier);
+	return 0;
 }
 
-int
-afr_sh_data_close (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t     *local      = NULL;
-        afr_private_t   *priv       = NULL;
-        afr_self_heal_t *sh         = NULL;
-        int              i          = 0;
-        int              call_count = 0;
-
-        local = frame->local;
-        sh    = &local->self_heal;
-        priv  = this->private;
-
-        call_count        = afr_set_elem_count_get (sh->success,
-                                                    priv->child_count);
-        local->call_count = call_count;
-
-        if (call_count == 0) {
-                afr_sh_data_done (frame, this);
-                return 0;
-        }
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (!sh->success[i])
-                        continue;
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "closing fd of %s on %s",
-                        local->loc.path, priv->children[i]->name);
-
-                STACK_WIND_COOKIE (frame, afr_sh_data_flush_cbk,
-                                   (void *) (long) i,
-                                   priv->children[i],
-                                   priv->children[i]->fops->flush,
-                                   sh->healing_fd, NULL);
-
-                if (!--call_count)
-                        break;
-        }
-
-        return 0;
-}
-
-int
-afr_sh_data_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, struct iatt *statpre,
-                         struct iatt *statpost, dict_t *xdata)
-{
-
-        afr_local_t   *local       = NULL;
-        afr_private_t *priv        = NULL;
-        int            call_count  = 0;
-        int            child_index = (long) cookie;
-
-        local = frame->local;
-        priv = this->private;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "setattr failed on %s on subvolume %s: %s",
-                                local->loc.path, priv->children[child_index]->name,
-                                strerror (op_errno));
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                afr_sh_data_finish (frame, this);
-        }
-
-        return 0;
-}
 
-int
-afr_sh_data_setattr (call_frame_t *frame, xlator_t *this, struct iatt* stbuf)
+static int
+attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+	  int op_ret, int op_errno, struct iatt *pre, struct iatt *post,
+	  dict_t *xdata)
 {
-        afr_local_t     *local      = NULL;
-        afr_private_t   *priv       = NULL;
-        afr_self_heal_t *sh         = NULL;
-        int              i          = 0;
-        int              call_count = 0;
-        int32_t          valid      = 0;
-
-        local = frame->local;
-        sh    = &local->self_heal;
-        priv  = this->private;
-
-        valid = (GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME);
-
-        call_count        = afr_set_elem_count_get (sh->success,
-                                                    priv->child_count);
-        local->call_count = call_count;
-
-        if (call_count == 0) {
-                GF_ASSERT (0);
-                afr_sh_data_finish (frame, this);
-                return 0;
-        }
+	int i = (long) cookie;
+	afr_local_t *local = NULL;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (!sh->success[i])
-                        continue;
+	local = frame->local;
 
-                STACK_WIND_COOKIE (frame, afr_sh_data_setattr_cbk,
-                                   (void *) (long) i,
-                                   priv->children[i],
-                                   priv->children[i]->fops->setattr,
-                                   &local->loc, stbuf, valid, NULL);
+	local->replies[i].valid = 1;
+	local->replies[i].op_ret = op_ret;
+	local->replies[i].op_errno = op_errno;
+	if (pre)
+		local->replies[i].prestat = *pre;
+	if (post)
+		local->replies[i].poststat = *post;
+	if (xdata)
+		local->replies[i].xdata = dict_ref (xdata);
 
-                if (!--call_count)
-                        break;
-        }
+	syncbarrier_wake (&local->barrier);
 
-        return 0;
+	return 0;
 }
 
-int
-afr_sh_data_setattr_fstat_cbk (call_frame_t *frame, void *cookie,
-                               xlator_t *this, int32_t op_ret, int32_t op_errno,
-                               struct iatt *buf, dict_t *xdata)
-{
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        int child_index = (long) cookie;
-
-        local = frame->local;
-        sh = &local->self_heal;
-
-        GF_ASSERT (sh->source == child_index);
-        if (op_ret != -1) {
-                sh->buf[child_index] = *buf;
-                afr_sh_data_setattr (frame, this, buf);
-        } else {
-                gf_log (this->name, GF_LOG_ERROR, "%s: Failed to set "
-                        "time-stamps after self-heal", local->loc.path);
-                afr_sh_data_fail (frame, this);
-        }
-
-        return 0;
-}
 
-/*
- * If there are any writes after the self-heal is triggered then the
- * stbuf stored in local->self_heal.buf[] will be invalid so we do one more
- * stat on the source and then set the [am]times
- */
-int
-afr_sh_set_timestamps (call_frame_t *frame, xlator_t *this)
+static gf_boolean_t
+__afr_selfheal_data_checksums_match (call_frame_t *frame, xlator_t *this,
+				     fd_t *fd, int source,
+				     unsigned char *healed_sinks,
+				     off_t offset, size_t size)
 {
-        afr_local_t     *local      = NULL;
-        afr_private_t   *priv       = NULL;
-        afr_self_heal_t *sh         = NULL;
-
-        local = frame->local;
-        sh    = &local->self_heal;
-        priv  = this->private;
-
-        STACK_WIND_COOKIE (frame, afr_sh_data_setattr_fstat_cbk,
-                           (void *) (long) sh->source,
-                           priv->children[sh->source],
-                           priv->children[sh->source]->fops->fstat,
-                           sh->healing_fd, NULL);
-        return 0;
-}
+	afr_private_t *priv = NULL;
+	afr_local_t *local = NULL;
+	unsigned char *wind_subvols = NULL;
+	int i = 0;
 
-//Fun fact, lock_cbk is being used for both lock & unlock
-int
-afr_sh_data_unlock (call_frame_t *frame, xlator_t *this,
-                    afr_lock_cbk_t lock_cbk)
-{
-        afr_local_t         *local    = NULL;
-        afr_internal_lock_t *int_lock = NULL;
-        afr_self_heal_t     *sh       = NULL;
+	priv = this->private;
+	local = frame->local;
 
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        sh       = &local->self_heal;
+	wind_subvols = alloca0 (priv->child_count);
+	for (i = 0; i < priv->child_count; i++) {
+		if (i == source || healed_sinks[i])
+			wind_subvols[i] = 1;
+	}
 
-        GF_ASSERT (sh->data_lock_held);
+	AFR_ONLIST (wind_subvols, frame, __checksum_cbk, rchecksum, fd,
+		    offset, size, NULL);
 
-        sh->data_lock_held = _gf_false;
-        int_lock->lock_cbk = lock_cbk;
-        afr_unlock (frame, this);
+	if (!local->replies[source].valid || local->replies[source].op_ret != 0)
+		return _gf_false;
 
-        return 0;
-}
+	for (i = 0; i < priv->child_count; i++) {
+		if (i == source)
+			continue;
+		if (memcmp (local->replies[source].checksum,
+			    local->replies[i].checksum,
+			    MD5_DIGEST_LENGTH))
+			return _gf_false;
+	}
 
-int
-afr_sh_data_finish (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t   *local = NULL;
-        afr_self_heal_t *sh = NULL;
-
-        local = frame->local;
-        sh = &local->self_heal;
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "finishing data selfheal of %s", local->loc.path);
-
-        if (sh->data_lock_held)
-                afr_sh_data_unlock (frame, this, afr_sh_data_close);
-        else
-                afr_sh_data_close (frame, this);
-
-        return 0;
+	return _gf_true;
 }
 
-int
-afr_sh_data_fail (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t   *local = NULL;
-        afr_self_heal_t *sh = NULL;
-
-        local = frame->local;
-        sh = &local->self_heal;
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "finishing failed data selfheal of %s", local->loc.path);
 
-        sh->op_failed = 1;
-        if (sh->data_lock_held)
-                afr_sh_data_unlock (frame, this, afr_sh_data_close);
-        else
-                afr_sh_data_close (frame, this);
-        return 0;
+static int
+__afr_selfheal_data_read_write (call_frame_t *frame, xlator_t *this, fd_t *fd,
+				int source, unsigned char *healed_sinks,
+				off_t offset, size_t size,
+				struct afr_reply *replies)
+{
+	struct iovec *iovec = NULL;
+	int count = 0;
+	struct iobref *iobref = NULL;
+	int ret = 0;
+	int i = 0;
+	afr_private_t *priv = NULL;
+
+	priv = this->private;
+
+	ret = syncop_readv (priv->children[source], fd, size, offset, 0,
+			    &iovec, &count, &iobref);
+	if (ret <= 0)
+		return ret;
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!healed_sinks[i])
+			continue;
+
+		/*
+		 * TODO: Use fiemap() and discard() to heal holes
+		 * in the future.
+		 *
+		 * For now,
+		 *
+		 * - if the source had any holes at all,
+		 * AND
+		 * - if we are writing past the original file size
+		 *   of the sink
+		 * AND
+		 * - is NOT the last block of the source file. if
+		 *   the block contains EOF, it has to be written
+		 *   in order to set the file size even if the
+		 *   last block is 0-filled.
+		 * AND
+		 * - if the read buffer is filled with only 0's
+		 *
+		 * then, skip writing to this source. We don't depend
+		 * on the write to happen to update the size as we
+		 * have performed an ftruncate() upfront anyways.
+		 */
+#define is_last_block(o,b,s) ((s >= o) && (s <= (o + b)))
+		if (HAS_HOLES ((&replies[source].poststat)) &&
+		    offset >= replies[i].poststat.ia_size &&
+		    !is_last_block (offset, size,
+				    replies[source].poststat.ia_size) &&
+		    (iov_0filled (iovec, count) == 0))
+			continue;
+
+		ret = syncop_writev (priv->children[i], fd, iovec, count,
+				     offset, iobref, 0);
+		if (ret != iov_length (iovec, count)) {
+			/* write() failed on this sink. unset the corresponding
+			   member in sinks[] (which is healed_sinks[] in the
+			   caller) so that this server does NOT get considered
+			   as successfully healed.
+			*/
+			healed_sinks[i] = 0;
+		}
+	}
+	if (iobref)
+		iobref_unref (iobref);
+
+	return ret;
 }
 
-int
-afr_sh_data_erase_pending_cbk (call_frame_t *frame, void *cookie,
-                               xlator_t *this, int32_t op_ret,
-                               int32_t op_errno, dict_t *xattr, dict_t *xdata)
-{
-        int             call_count = 0;
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        int32_t         child_index = (long) cookie;
-
-        priv  = this->private;
-        local = frame->local;
-        sh    = &local->self_heal;
-        if (op_ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Erasing of pending change "
-                        "log failed on %s for subvol %s, reason: %s",
-                        local->loc.path, priv->children[child_index]->name,
-                        strerror (op_errno));
-                sh->op_failed = 1;
-        }
-
-        call_count = afr_frame_return (frame);
 
-        if (call_count == 0) {
-                if (sh->op_failed) {
-                        if (sh->old_loop_frame)
-                                sh_loop_finish (sh->old_loop_frame, this);
-                        sh->old_loop_frame = NULL;
-                        afr_sh_data_fail (frame, this);
-                        goto out;
-                }
-                if (!IA_ISREG (sh->type)) {
-                        afr_sh_data_finish (frame, this);
-                        goto out;
-                }
-                GF_ASSERT (sh->old_loop_frame);
-                afr_sh_data_lock (frame, this, 0, 0,
-                                  afr_post_sh_big_lock_success,
-                                  afr_post_sh_big_lock_failure);
-        }
-out:
-        return 0;
+static int
+afr_selfheal_data_block (call_frame_t *frame, xlator_t *this, fd_t *fd,
+			 int source, unsigned char *healed_sinks, off_t offset,
+			 size_t size, int type, struct afr_reply *replies)
+{
+	int ret = -1;
+	int sink_count = 0;
+	afr_private_t *priv = NULL;
+	unsigned char *data_lock = NULL;
+
+	priv = this->private;
+	sink_count = AFR_COUNT (healed_sinks, priv->child_count);
+	data_lock = alloca0 (priv->child_count);
+
+	ret = afr_selfheal_inodelk (frame, this, fd->inode, this->name,
+				    offset, size, data_lock);
+	{
+		if (ret < sink_count) {
+			ret = -ENOTCONN;
+			goto unlock;
+		}
+
+		if (type == AFR_SELFHEAL_DATA_DIFF &&
+		    __afr_selfheal_data_checksums_match (frame, this, fd, source,
+							 healed_sinks, offset, size)) {
+			ret = 0;
+			goto unlock;
+		}
+
+		ret = __afr_selfheal_data_read_write (frame, this, fd, source,
+						      healed_sinks, offset, size,
+						      replies);
+	}
+unlock:
+	afr_selfheal_uninodelk (frame, this, fd->inode, this->name,
+				offset, size, data_lock);
+	return ret;
 }
 
-int
-afr_sh_data_erase_pending (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        int              call_count = 0;
-        int              i = 0;
-        dict_t          **erase_xattr = NULL;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        afr_sh_pending_to_delta (priv, sh->xattr, sh->delta_matrix, sh->success,
-                                 priv->child_count, AFR_DATA_TRANSACTION);
-        gf_log (this->name, GF_LOG_DEBUG, "Delta matrix for: %s",
-                lkowner_utoa (&frame->root->lk_owner));
-        afr_sh_print_pending_matrix (sh->delta_matrix, this);
-
-        erase_xattr = GF_CALLOC (sizeof (*erase_xattr), priv->child_count,
-                                 gf_afr_mt_dict_t);
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (sh->xattr[i]) {
-                        call_count++;
-
-                        erase_xattr[i] = get_new_dict();
-                        dict_ref (erase_xattr[i]);
-                }
-        }
 
-        afr_sh_delta_to_xattr (priv, sh->delta_matrix, erase_xattr,
-                               priv->child_count, AFR_DATA_TRANSACTION);
-
-        GF_ASSERT (call_count);
-        local->call_count = call_count;
-        for (i = 0; i < priv->child_count; i++) {
-                if (!erase_xattr[i])
-                        continue;
-
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "erasing pending flags from %s on %s",
-                        local->loc.path, priv->children[i]->name);
-
-                STACK_WIND_COOKIE (frame, afr_sh_data_erase_pending_cbk,
-                                   (void *) (long) i,
-                                   priv->children[i],
-                                   priv->children[i]->fops->fxattrop,
-                                   sh->healing_fd,
-                                   GF_XATTROP_ADD_ARRAY, erase_xattr[i],
-                                   NULL);
-                if (!--call_count)
-                        break;
-        }
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (erase_xattr[i]) {
-                        dict_unref (erase_xattr[i]);
-                }
-        }
-        GF_FREE (erase_xattr);
 
-        return 0;
-}
-
-
-static struct afr_sh_algorithm *
-sh_algo_from_name (xlator_t *this, char *name)
+static int
+afr_selfheal_data_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd,
+			 unsigned char *healed_sinks)
 {
-        int i = 0;
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	int i = 0;
 
-        while (afr_self_heal_algorithms[i].name) {
-                if (!strcmp (name, afr_self_heal_algorithms[i].name)) {
-                        return &afr_self_heal_algorithms[i];
-                }
+	local = frame->local;
+	priv = this->private;
 
-                i++;
-        }
+	AFR_ONLIST (healed_sinks, frame, attr_cbk, fsync, fd, 0, NULL);
 
-        return NULL;
+	for (i = 0; i < priv->child_count; i++)
+		if (healed_sinks[i] && local->replies[i].op_ret != 0)
+			/* fsync() failed. Do NOT consider this server
+			   as successfully healed. Mark it so.
+			*/
+			healed_sinks[i] = 0;
+	return 0;
 }
 
 
 static int
-sh_zero_byte_files_exist (afr_local_t *local, int child_count)
+afr_selfheal_data_restore_time (call_frame_t *frame, xlator_t *this,
+				inode_t *inode, int source,
+				unsigned char *healed_sinks,
+				struct afr_reply *replies)
 {
-        int             i = 0;
-        int             ret = 0;
-        afr_self_heal_t *sh = NULL;
-
-        sh = &local->self_heal;
-        for (i = 0; i < child_count; i++) {
-                if (!local->child_up[i] || sh->child_errno[i])
-                        continue;
-                if (sh->buf[i].ia_size == 0) {
-                        ret = 1;
-                        break;
-                }
-        }
+	loc_t loc = {0, };
 
-        return ret;
-}
+	loc.inode = inode_ref (inode);
+	uuid_copy (loc.gfid, inode->gfid);
 
+	AFR_ONLIST (healed_sinks, frame, attr_cbk, setattr, &loc,
+		    &replies[source].poststat,
+		    (GF_SET_ATTR_ATIME|GF_SET_ATTR_MTIME), NULL);
 
-struct afr_sh_algorithm *
-afr_sh_data_pick_algo (call_frame_t *frame, xlator_t *this)
-{
-        afr_private_t *           priv  = NULL;
-        struct afr_sh_algorithm * algo  = NULL;
-        afr_local_t *             local = NULL;
-        afr_self_heal_t *         sh    = NULL;
-
-        priv  = this->private;
-        local = frame->local;
-        sh    = &local->self_heal;
-        algo  = sh_algo_from_name (this, priv->data_self_heal_algorithm);
-
-        if (algo == NULL) {
-                /* option not set, so fall back on heuristics */
-
-                if (sh_zero_byte_files_exist (local, priv->child_count)
-                    || (sh->file_size <= (priv->data_self_heal_window_size *
-                                          this->ctx->page_size))) {
-
-                        /*
-                         * If the file does not exist on one of the subvolumes,
-                         * or a zero-byte file exists (created by entry self-heal)
-                         * the entire content has to be copied anyway, so there
-                         * is no benefit from using the "diff" algorithm.
-                         *
-                         * If the file size is about the same as page size,
-                         * the entire file can be read and written with a few
-                         * (pipelined) STACK_WINDs, which will be faster
-                         * than "diff" which has to read checksums and then
-                         * read and write.
-                         */
-
-                        algo = sh_algo_from_name (this, "full");
-
-                } else {
-                        algo = sh_algo_from_name (this, "diff");
-                }
-        }
+	loc_wipe (&loc);
 
-        return algo;
+	return 0;
 }
 
-
-int
-afr_sh_data_sync_prepare (call_frame_t *frame, xlator_t *this)
+static int
+afr_data_self_heal_type_get (afr_private_t *priv, unsigned char *healed_sinks,
+                             int source, struct afr_reply *replies)
 {
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        struct afr_sh_algorithm *sh_algo = NULL;
-
-        local = frame->local;
-        sh = &local->self_heal;
-
-        sh->algo_completion_cbk = afr_sh_data_erase_pending;
-        sh->algo_abort_cbk      = afr_sh_data_fail;
-
-        sh_algo = afr_sh_data_pick_algo (frame, this);
-
-        sh->algo = sh_algo;
-        sh_algo->fn (frame, this);
-
-        return 0;
-}
+        int type = AFR_SELFHEAL_DATA_FULL;
+        int i = 0;
 
-int
-afr_sh_data_trim_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                      struct iatt *postbuf, dict_t *xdata)
-{
-        int              call_count = 0;
-        int              child_index = 0;
-        afr_private_t    *priv = NULL;
-        afr_local_t      *local  = NULL;
-        afr_self_heal_t  *sh = NULL;
-
-        priv  = this->private;
-        local = frame->local;
-        sh    = &local->self_heal;
-
-        child_index = (long) cookie;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "ftruncate of %s on subvolume %s failed (%s)",
-                                local->loc.path,
-                                priv->children[child_index]->name,
-                                strerror (op_errno));
-                        sh->op_failed = 1;
-                } else {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "ftruncate of %s on subvolume %s completed",
-                                local->loc.path,
-                                priv->children[child_index]->name);
+        if (priv->data_self_heal_algorithm == NULL) {
+                type = AFR_SELFHEAL_DATA_FULL;
+                for (i = 0; i < priv->child_count; i++) {
+                        if (!healed_sinks[i] && i != source)
+                                continue;
+                        if (replies[i].poststat.ia_size) {
+                                type = AFR_SELFHEAL_DATA_DIFF;
+                                break;
+                        }
                 }
+        } else if (strcmp (priv->data_self_heal_algorithm, "full") == 0) {
+                type = AFR_SELFHEAL_DATA_FULL;
+        } else if (strcmp (priv->data_self_heal_algorithm, "diff") == 0) {
+                type = AFR_SELFHEAL_DATA_DIFF;
         }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                if (sh->op_failed)
-                        afr_sh_data_fail (frame, this);
-                else
-                        afr_sh_data_sync_prepare (frame, this);
-        }
-
-        return 0;
-}
-
-
-int
-afr_sh_data_trim_sinks (call_frame_t *frame, xlator_t *this)
-{
-        afr_private_t * priv = NULL;
-        afr_local_t * local  = NULL;
-        afr_self_heal_t *sh  = NULL;
-        int             *sources = NULL;
-        int              call_count = 0;
-        int              i = 0;
-
-
-        priv = this->private;
-        local = frame->local;
-        sh = &local->self_heal;
-
-        sources = sh->sources;
-        call_count = sh->active_sinks;
-
-        local->call_count = call_count;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (sources[i] || !local->child_up[i])
-                        continue;
-
-                STACK_WIND_COOKIE (frame, afr_sh_data_trim_cbk,
-                                   (void *) (long) i,
-                                   priv->children[i],
-                                   priv->children[i]->fops->ftruncate,
-                                   sh->healing_fd, sh->file_size,
-                                   NULL);
-
-                if (!--call_count)
-                        break;
-        }
-
-        return 0;
+        return type;
 }
 
-int
-afr_sh_inode_set_read_ctx (afr_self_heal_t *sh, xlator_t *this)
-{
-        afr_private_t   *priv = NULL;
-        int             ret = 0;
-        int             i = 0;
-
-        priv = this->private;
-        sh->source = afr_sh_select_source (sh->sources, priv->child_count);
-        if (sh->source < 0) {
-                ret = -1;
-                goto out;
-        }
-
-        /* detect changes not visible through pending flags -- JIC */
-        for (i = 0; i < priv->child_count; i++) {
-                if (i == sh->source || sh->child_errno[i])
-                        continue;
-
-                if (SIZE_DIFFERS (&sh->buf[i], &sh->buf[sh->source]))
-                        sh->sources[i] = 0;
-        }
+static int
+afr_selfheal_data_do (call_frame_t *frame, xlator_t *this, fd_t *fd,
+		      int source, unsigned char *healed_sinks,
+		      struct afr_reply *replies)
+{
+	afr_private_t *priv = NULL;
+	int i = 0;
+	off_t off = 0;
+	size_t block = 128 * 1024;
+	int type = AFR_SELFHEAL_DATA_FULL;
+	int ret = -1;
+	call_frame_t *iter_frame = NULL;
+	char *sinks_str = NULL;
+	char *p = NULL;
+
+	priv = this->private;
+
+	sinks_str = alloca0 (priv->child_count * 8);
+	p = sinks_str;
+	for (i = 0; i < priv->child_count; i++) {
+		if (!healed_sinks[i])
+			continue;
+		p += sprintf (p, "%d ", i);
+	}
+
+	gf_log (this->name, GF_LOG_INFO, "performing data selfheal on %s. "
+		"source=%d sinks=%s",
+		uuid_utoa (fd->inode->gfid), source, sinks_str);
+
+        type = afr_data_self_heal_type_get (priv, healed_sinks, source,
+                                            replies);
+
+	iter_frame = afr_copy_frame (frame);
+	if (!iter_frame)
+		return -ENOMEM;
+
+	for (off = 0; off < replies[source].poststat.ia_size; off += block) {
+		ret = afr_selfheal_data_block (iter_frame, this, fd, source,
+					       healed_sinks, off, block, type,
+					       replies);
+		if (ret < 0)
+			goto out;
+
+		AFR_STACK_RESET (iter_frame);
+	}
+
+	afr_selfheal_data_restore_time (frame, this, fd->inode, source,
+					healed_sinks, replies);
+
+	ret = afr_selfheal_data_fsync (frame, this, fd, healed_sinks);
 
-        afr_reset_children (sh->fresh_children, priv->child_count);
-        afr_get_fresh_children (sh->success_children, sh->sources,
-                                sh->fresh_children, priv->child_count);
-        afr_inode_set_read_ctx (this, sh->inode, sh->source,
-                                sh->fresh_children);
 out:
-        return ret;
+	if (iter_frame)
+		AFR_STACK_DESTROY (iter_frame);
+	return ret;
 }
 
-void
-afr_sh_data_fix (call_frame_t *frame, xlator_t *this)
-{
-        int              source = 0;
-        afr_local_t     *local      = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        source     = sh->source;
-        sh->block_size = this->ctx->page_size;
-        sh->file_size  = sh->buf[source].ia_size;
-
-        if (FILE_HAS_HOLES (&sh->buf[source]))
-                sh->file_has_holes = 1;
-
-        if (sh->background && sh->unwind && !sh->unwound) {
-                sh->unwind (sh->orig_frame, this, sh->op_ret, sh->op_errno,
-                            sh->op_failed);
-                sh->unwound = _gf_true;
-        }
-
-        afr_sh_mark_source_sinks (frame, this);
-        if (sh->active_sinks == 0) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "no active sinks for performing self-heal on file %s",
-                        local->loc.path);
-                afr_sh_data_finish (frame, this);
-                return;
-        }
 
-        gf_log (this->name, GF_LOG_DEBUG,
-                "self-healing file %s from subvolume %s to %d other",
-                local->loc.path, priv->children[sh->source]->name,
-                sh->active_sinks);
-        afr_sh_data_trim_sinks (frame, this);
-}
-
-int
-afr_sh_data_fxattrop_fstat_done (call_frame_t *frame, xlator_t *this)
+static int
+__afr_selfheal_truncate_sinks (call_frame_t *frame, xlator_t *this,
+			       fd_t *fd, unsigned char *healed_sinks,
+			       struct afr_reply *replies, uint64_t size)
 {
-        afr_local_t     *local      = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        int              nsources = 0;
-        int              ret = 0;
-        int             *old_sources = NULL;
-        int             tstamp_source = 0;
-        int             i = 0;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        gf_log (this->name, GF_LOG_DEBUG, "Pending matrix for: %s",
-                lkowner_utoa (&frame->root->lk_owner));
-        if (sh->sync_done) {
-                //store sources before sync so that mtime can be set using the
-                //iatt buf from one of them.
-                old_sources = alloca (priv->child_count*sizeof (*old_sources));
-                memcpy (old_sources, sh->sources,
-                        priv->child_count * sizeof (*old_sources));
-        }
-
-        nsources = afr_build_sources (this, sh->xattr, sh->buf, sh->pending_matrix,
-                                      sh->sources, sh->success_children,
-                                      AFR_DATA_TRANSACTION, NULL, _gf_true);
-        if ((nsources == -1)
-            && (priv->favorite_child != -1)
-            && (sh->child_errno[priv->favorite_child] == 0)) {
-
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Picking favorite child %s as authentic source to "
-                        "resolve conflicting data of %s",
-                        priv->children[priv->favorite_child]->name,
-                        local->loc.path);
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	unsigned char *larger_sinks = 0;
+	int i = 0;
 
-                sh->sources[priv->favorite_child] = 1;
-
-                nsources = afr_sh_source_count (sh->sources,
-                                                priv->child_count);
-        }
+	local = frame->local;
+	priv = this->private;
 
-        if (nsources == -1) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to self-heal contents of '%s' (possible "
-                        "split-brain). Please delete the file from all but "
-                        "the preferred subvolume.", local->loc.path);
+	larger_sinks = alloca0 (priv->child_count);
+	for (i = 0; i < priv->child_count; i++) {
+		if (healed_sinks[i] && replies[i].poststat.ia_size > size)
+			larger_sinks[i] = 1;
+	}
 
-                sh->data_spb = _gf_true;
-                afr_sh_data_fail (frame, this);
-                return 0;
-        }
-
-        sh->data_spb = _gf_false;
-        ret = afr_sh_inode_set_read_ctx (sh, this);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "No active sources found.");
-
-                afr_sh_data_fail (frame, this);
-                return 0;
-        }
+	AFR_ONLIST (larger_sinks, frame, attr_cbk, ftruncate, fd, size, NULL);
 
-        if (sh->sync_done) {
-                /* Perform setattr from one of the old_sources if possible
-                 * Because only they have the correct mtime, the new sources
-                 * (i.e. old sinks) have mtime from last writev in sync.
-                 */
-                tstamp_source = sh->source;
-                for (i = 0; i < priv->child_count; i++) {
-                        if (old_sources[i] && sh->sources[i])
-                                tstamp_source = i;
-                }
-                afr_sh_data_setattr (frame, this, &sh->buf[tstamp_source]);
-        } else {
-                if (nsources == 0) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "No self-heal needed for %s",
-                                local->loc.path);
-
-                        afr_sh_data_finish (frame, this);
-                        return 0;
-                }
-
-                afr_sh_data_fix (frame, this);
-        }
-        return 0;
+	for (i = 0; i < priv->child_count; i++)
+		if (healed_sinks[i] && local->replies[i].op_ret == -1)
+			/* truncate() failed. Do NOT consider this server
+			   as successfully healed. Mark it so.
+			*/
+			healed_sinks[i] = 0;
+	return 0;
 }
 
-int
-afr_lookup_select_read_child_by_txn_type (xlator_t *this, afr_local_t *local,
-                                          dict_t **xattr,
-                                          afr_transaction_type txn_type)
-{
-        afr_private_t            *priv      = NULL;
-        int                      read_child = -1;
-        int32_t                  **pending_matrix = NULL;
-        int32_t                  *sources         = NULL;
-        int32_t                  *success_children   = NULL;
-        struct iatt              *bufs            = NULL;
-        int32_t                  nsources         = 0;
-        int32_t                  prev_read_child  = -1;
-        int32_t                  config_read_child = -1;
-        int32_t                  subvol_status = 0;
-
-        priv = this->private;
-        bufs = local->cont.lookup.bufs;
-        success_children = local->cont.lookup.success_children;
-
-        pending_matrix = afr_matrix_create (priv->child_count,
-                                            priv->child_count);
-        if (NULL == pending_matrix)
-                goto out;
-
-        sources = local->cont.lookup.sources;
-        memset (sources, 0, sizeof (*sources) * priv->child_count);
-
-        nsources = afr_build_sources (this, xattr, bufs, pending_matrix,
-                                      sources, success_children, txn_type,
-                                      &subvol_status, _gf_false);
-        if (subvol_status & SPLIT_BRAIN) {
-                gf_log (this->name, GF_LOG_DEBUG, "%s: Possible split-brain",
-                        local->loc.path);
-                switch (txn_type) {
-                case AFR_DATA_TRANSACTION:
-                        local->cont.lookup.possible_spb = _gf_true;
-                        nsources = 1;
-                        sources[success_children[0]] = 1;
-                        break;
-                case AFR_ENTRY_TRANSACTION:
-                        read_child = afr_get_no_xattr_dir_read_child (this,
-                                                             success_children,
-                                                             bufs);
-                        sources[read_child] = 1;
-                        nsources = 1;
-                        break;
-                default:
-                        break;
-                }
-        }
-        if (nsources < 0)
-                goto out;
-
-        prev_read_child = local->read_child_index;
-        config_read_child = priv->read_child;
-        read_child = afr_select_read_child_from_policy (success_children,
-                                                        priv->child_count,
-                                                        prev_read_child,
-                                                        config_read_child,
-                                                        sources);
-out:
-        afr_matrix_cleanup (pending_matrix, priv->child_count);
-        gf_log (this->name, GF_LOG_DEBUG, "returning read_child: %d",
-                read_child);
-        return read_child;
+/*
+ * If by chance there are multiple sources with differing sizes, select
+ * the largest file as the source.
+ *
+ * This can only happen if data was directly modified in the backend.
+ */
+static int
+__afr_selfheal_data_finalize_source (xlator_t *this, unsigned char *sources,
+				     unsigned char *sinks,
+				     unsigned char *healed_sinks,
+				     unsigned char *locked_on,
+				     struct afr_reply *replies)
+{
+	int i = 0;
+	afr_private_t *priv = NULL;
+	uint64_t size = 0;
+	int source = -1;
+	int locked_count = 0;
+	int sources_count = 0;
+	int healed_sinks_count = 0;
+
+	priv = this->private;
+
+	locked_count = AFR_COUNT (locked_on, priv->child_count);
+	sources_count = AFR_COUNT (sources, priv->child_count);
+	healed_sinks_count = AFR_COUNT (healed_sinks, priv->child_count);
+
+	if (locked_count == healed_sinks_count || !sources_count) {
+		/* split brain */
+		return -EIO;
+	}
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!sources[i])
+			continue;
+		if (size <= replies[i].poststat.ia_size) {
+			size = replies[i].poststat.ia_size;
+			source = i;
+		}
+	}
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!sources[i])
+			continue;
+		if (replies[i].poststat.ia_size < size) {
+			sources[i] = 0;
+			sinks[i] = 1;
+		}
+	}
+
+	return source;
 }
 
-int
-afr_sh_data_special_file_fix (call_frame_t *frame, xlator_t *this)
+/*
+ * __afr_selfheal_data_prepare:
+ *
+ * This function inspects the on-disk xattrs and determines which subvols
+ * are sources and sinks.
+ *
+ * The return value is the index of the subvolume to be used as the source
+ * for self-healing, or -1 if no healing is necessary/split brain.
+ */
+static int
+__afr_selfheal_data_prepare (call_frame_t *frame, xlator_t *this, fd_t *fd,
+			     unsigned char *locked_on, unsigned char *sources,
+			     unsigned char *sinks, unsigned char *healed_sinks,
+			     struct afr_reply *replies)
 {
-        afr_private_t   *priv = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_local_t     *local = NULL;
-        int             i = 0;
+	int ret = -1;
+	int source = -1;
+	afr_private_t *priv = NULL;
+	int i = 0;
 
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
+	priv = this->private;
 
-        for (i = 0; i < priv->child_count ; i++)
-                if (1 == local->child_up[i])
-                        sh->success[i] = 1;
+	ret = afr_selfheal_unlocked_discover (frame, fd->inode, fd->inode->gfid,
+					      replies);
+	if (ret)
+		return ret;
 
-        afr_sh_data_erase_pending (frame, this);
+	ret = afr_selfheal_find_direction (frame, this, replies,
+					   AFR_DATA_TRANSACTION,
+					   locked_on, sources, sinks);
+	if (ret)
+		return ret;
 
-        return 0;
-}
+	source = __afr_selfheal_data_finalize_source (this, sources, sinks,
+						      healed_sinks, locked_on,
+						      replies);
+	if (source < 0)
+		return -EIO;
 
-int
-afr_sh_data_fstat_cbk (call_frame_t *frame, void *cookie,
-                       xlator_t *this, int32_t op_ret, int32_t op_errno,
-                       struct iatt *buf, dict_t *xdata)
-{
-        afr_private_t   *priv  = NULL;
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        int call_count  = -1;
-        int child_index = (long) cookie;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret != -1) {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "fstat of %s on %s succeeded",
-                                local->loc.path,
-                                priv->children[child_index]->name);
-
-                        sh->buf[child_index] = *buf;
-                        sh->success_children[sh->success_count] = child_index;
-                        sh->success_count++;
-                } else {
-                        gf_log (this->name, GF_LOG_ERROR, "%s: fstat failed "
-                                "on %s, reason %s", local->loc.path,
-                                priv->children[child_index]->name,
-                                strerror (op_errno));
-                        sh->child_errno[child_index] = op_errno;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                /* Previous versions of glusterfs might have set
-                 * the pending data xattrs which need to be erased
-                 */
-                if (!afr_sh_data_proceed (sh->success_count)) {
-                        gf_log (this->name, GF_LOG_ERROR, "inspecting metadata "
-                                "succeeded on < %d children, aborting "
-                                "self-heal for %s", AFR_SH_MIN_PARTICIPANTS,
-                                local->loc.path);
-                        afr_sh_data_fail (frame, this);
-                        goto out;
-                }
-                if (IA_ISREG (sh->type))
-                        afr_sh_data_fxattrop_fstat_done (frame, this);
-                else
-                        afr_sh_data_special_file_fix (frame, this);
-        }
-out:
-        return 0;
-}
+	for (i = 0; i < priv->child_count; i++)
+		/* Initialize the healed_sinks[] array optimistically to
+		   the intersection of to-be-healed (i.e sinks[]) and
+		   the list of servers which are up (i.e locked_on[]).
 
+		   As we encounter failures in the healing process, we
+		   will unmark the respective servers in the healed_sinks[]
+		   array.
+		*/
+		healed_sinks[i] = sinks[i] && locked_on[i];
 
-int
-afr_sh_data_fstat (call_frame_t *frame, xlator_t *this)
-{
-        afr_self_heal_t *sh    = NULL;
-        afr_local_t     *local = NULL;
-        afr_private_t   *priv  = NULL;
-        int             call_count = 0;
-        int             i = 0;
-        int             child = 0;
-        int32_t         *fstat_children = NULL;
-
-        priv  = this->private;
-        local = frame->local;
-        sh    = &local->self_heal;
-
-        fstat_children = memdup (sh->success_children,
-                                 sizeof (*fstat_children) * priv->child_count);
-        if (!fstat_children) {
-                afr_sh_data_fail (frame, this);
-                goto out;
-        }
-        call_count = sh->success_count;
-        local->call_count = call_count;
-
-        memset (sh->buf, 0, sizeof (*sh->buf) * priv->child_count);
-        afr_reset_children (sh->success_children, priv->child_count);
-        sh->success_count = 0;
-        for (i = 0; i < priv->child_count; i++) {
-                child = fstat_children[i];
-                if (child == -1)
-                        break;
-                STACK_WIND_COOKIE (frame, afr_sh_data_fstat_cbk,
-                                   (void *) (long) child,
-                                   priv->children[child],
-                                   priv->children[child]->fops->fstat,
-                                   sh->healing_fd, NULL);
-                --call_count;
-        }
-        GF_ASSERT (!call_count);
-out:
-        if (fstat_children)
-                GF_FREE (fstat_children);
-        return 0;
+	return source;
 }
 
-void
-afr_sh_common_fxattrop_resp_handler (call_frame_t *frame, void *cookie,
-                                     xlator_t *this, int32_t op_ret,
-                                     int32_t op_errno, dict_t *xattr)
-{
-        afr_private_t   *priv  = NULL;
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        int child_index = (long) cookie;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret != -1) {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "fxattrop of %s on %s succeeded",
-                                local->loc.path,
-                                priv->children[child_index]->name);
-
-                        sh->xattr[child_index] = dict_ref (xattr);
-                        sh->success_children[sh->success_count] = child_index;
-                        sh->success_count++;
-                } else {
-                        gf_log (this->name, GF_LOG_ERROR, "fxattrop of %s "
-                                "failed on %s, reason %s", local->loc.path,
-                                priv->children[child_index]->name,
-                                strerror (op_errno));
-                        sh->child_errno[child_index] = op_errno;
-                }
-        }
-        UNLOCK (&frame->lock);
-}
-
-int
-afr_sh_data_fxattrop_cbk (call_frame_t *frame, void *cookie,
-                          xlator_t *this, int32_t op_ret, int32_t op_errno,
-                          dict_t *xattr, dict_t *xdata)
-{
-        int             call_count  = -1;
-        afr_local_t     *local  = NULL;
-        afr_self_heal_t *sh     = NULL;
-
-        local = frame->local;
-        sh    = &local->self_heal;
-
-        afr_sh_common_fxattrop_resp_handler (frame, cookie, this, op_ret,
-                                             op_errno, xattr);
-
-        call_count = afr_frame_return (frame);
-        if (call_count == 0) {
-                if (!afr_sh_data_proceed (sh->success_count)) {
-                        gf_log (this->name, GF_LOG_ERROR, "%s, inspecting "
-                                "change log succeeded on < %d children",
-                                local->loc.path, AFR_SH_MIN_PARTICIPANTS);
-                        afr_sh_data_fail (frame, this);
-                        goto out;
-                }
-                afr_sh_data_fstat (frame, this);
-        }
-out:
-        return 0;
-}
-
-
-int
-afr_sh_data_fxattrop (call_frame_t *frame, xlator_t *this)
-{
-        afr_self_heal_t *sh    = NULL;
-        afr_local_t     *local = NULL;
-        afr_private_t   *priv  = NULL;
-        dict_t          *xattr_req = NULL;
-        int32_t         *zero_pending = NULL;
-        int call_count = 0;
-        int i = 0;
-        int ret = 0;
-
-        priv  = this->private;
-        local = frame->local;
-        sh    = &local->self_heal;
-
-        call_count = afr_up_children_count (local->child_up,
-                                            priv->child_count);
-
-        local->call_count = call_count;
-
-        xattr_req = dict_new();
-        if (!xattr_req) {
-                ret = -1;
-                goto out;
-        }
-
-        for (i = 0; i < priv->child_count; i++) {
-                zero_pending = GF_CALLOC (3, sizeof (*zero_pending),
-                                          gf_afr_mt_int32_t);
-                if (!zero_pending) {
-                        ret = -1;
-                        goto out;
-                }
-                ret = dict_set_dynptr (xattr_req, priv->pending_key[i],
-                                       zero_pending,
-                                       3 * sizeof (*zero_pending));
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "Unable to set dict value");
-                        goto out;
-                } else {
-                        zero_pending = NULL;
-                }
-        }
-
-        afr_reset_xattr (sh->xattr, priv->child_count);
-        afr_reset_children (sh->success_children, priv->child_count);
-        memset (sh->child_errno, 0,
-                sizeof (*sh->child_errno) * priv->child_count);
-        sh->success_count = 0;
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, afr_sh_data_fxattrop_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->fxattrop,
-                                           sh->healing_fd, GF_XATTROP_ADD_ARRAY,
-                                           xattr_req, NULL);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
 
+static int
+__afr_selfheal_data (call_frame_t *frame, xlator_t *this, fd_t *fd,
+		     unsigned char *locked_on)
+{
+	afr_private_t *priv = NULL;
+	int ret = -1;
+	unsigned char *sources = NULL;
+	unsigned char *sinks = NULL;
+	unsigned char *data_lock = NULL;
+	unsigned char *healed_sinks = NULL;
+	struct afr_reply *locked_replies = NULL;
+	int source = -1;
+	gf_boolean_t compat = _gf_false;
+	unsigned char *compat_lock = NULL;
+
+	priv = this->private;
+
+	sources = alloca0 (priv->child_count);
+	sinks = alloca0 (priv->child_count);
+	healed_sinks = alloca0 (priv->child_count);
+	data_lock = alloca0 (priv->child_count);
+	compat_lock = alloca0 (priv->child_count);
+
+	locked_replies = alloca0 (sizeof (*locked_replies) * priv->child_count);
+
+	ret = afr_selfheal_inodelk (frame, this, fd->inode, this->name, 0, 0,
+				    data_lock);
+	{
+		if (ret < 2) {
+			ret = -ENOTCONN;
+			goto unlock;
+		}
+
+		ret = __afr_selfheal_data_prepare (frame, this, fd, data_lock,
+						   sources, sinks, healed_sinks,
+						   locked_replies);
+		if (ret < 0)
+			goto unlock;
+
+		source = ret;
+
+		ret = __afr_selfheal_truncate_sinks (frame, this, fd, healed_sinks,
+						     locked_replies,
+						     locked_replies[source].poststat.ia_size);
+		if (ret < 0)
+			goto unlock;
+
+		ret = 0;
+
+		/* Locking from (LLONG_MAX - 2) to (LLONG_MAX - 1) is for
+		   compatibility with older self-heal clients which do not
+		   hold a lock in the @priv->sh_domain domain to guard
+		   against concurrent ongoing self-heals
+		*/
+		afr_selfheal_inodelk (frame, this, fd->inode, this->name,
+				      LLONG_MAX - 2, 1, compat_lock);
+		compat = _gf_true;
+	}
+unlock:
+	afr_selfheal_uninodelk (frame, this, fd->inode, this->name, 0, 0,
+				data_lock);
+	if (ret < 0)
+		goto out;
+
+	ret = afr_selfheal_data_do (frame, this, fd, source, healed_sinks,
+				    locked_replies);
+	if (ret)
+		goto out;
+
+	ret = afr_selfheal_undo_pending (frame, this, fd->inode, sources, sinks,
+					 healed_sinks, AFR_DATA_TRANSACTION,
+					 locked_replies, data_lock);
 out:
-        if (xattr_req)
-                dict_unref (xattr_req);
-
-        if (ret) {
-                if (zero_pending)
-                        GF_FREE (zero_pending);
-                afr_sh_data_fail (frame, this);
-        }
-
-        return 0;
+	if (compat)
+		afr_selfheal_uninodelk (frame, this, fd->inode, this->name,
+					LLONG_MAX - 2, 1, compat_lock);
+	return ret;
 }
 
-int
-afr_sh_data_big_lock_success (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t   *local = NULL;
-        afr_self_heal_t *sh = NULL;
 
-        local = frame->local;
-        sh = &local->self_heal;
-
-        sh->data_lock_held = _gf_true;
-        afr_sh_data_fxattrop (frame, this);
-        return 0;
-}
-
-int
-afr_sh_data_post_blocking_inodelk_cbk (call_frame_t *frame, xlator_t *this)
+static fd_t *
+afr_selfheal_data_open (xlator_t *this, inode_t *inode)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_self_heal_t     *sh       = NULL;
+	loc_t loc = {0,};
+	int ret = 0;
+	fd_t *fd = NULL;
 
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        sh       = &local->self_heal;
+	fd = fd_create (inode, 0);
+	if (!fd)
+		return NULL;
 
-        if (int_lock->lock_op_ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Blocking data inodelks "
-                        "failed for %s. by %s",
-                        local->loc.path, lkowner_utoa (&frame->root->lk_owner));
+	loc.inode = inode_ref (inode);
+	uuid_copy (loc.gfid, inode->gfid);
 
-                sh->data_lock_failure_handler (frame, this);
-        } else {
+	ret = syncop_open (this, &loc, O_RDWR|O_LARGEFILE, fd);
+	if (ret) {
+		fd_unref (fd);
+		fd = NULL;
+	} else {
+		fd_bind (fd);
+	}
 
-                gf_log (this->name, GF_LOG_DEBUG, "Blocking data inodelks "
-                        "done for %s by %s. Proceding to self-heal",
-                        local->loc.path, lkowner_utoa (&frame->root->lk_owner));
+	loc_wipe (&loc);
 
-                sh->data_lock_success_handler (frame, this);
-        }
-
-        return 0;
+	return fd;
 }
 
-int
-afr_sh_data_post_nonblocking_inodelk_cbk (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_self_heal_t     *sh       = NULL;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        sh       = &local->self_heal;
-
-        if (int_lock->lock_op_ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG, "Non Blocking data inodelks "
-                        "failed for %s. by %s",
-                        local->loc.path, lkowner_utoa (&frame->root->lk_owner));
-
-                int_lock->lock_cbk = afr_sh_data_post_blocking_inodelk_cbk;
-                afr_blocking_lock (frame, this);
-        } else {
-
-                gf_log (this->name, GF_LOG_DEBUG, "Non Blocking data inodelks "
-                        "done for %s by %s. Proceeding to self-heal",
-                        local->loc.path, lkowner_utoa (&frame->root->lk_owner));
-                sh->data_lock_success_handler (frame, this);
-        }
-
-        return 0;
-}
 
 int
-afr_sh_data_lock_rec (call_frame_t *frame, xlator_t *this, off_t start, off_t len)
+afr_selfheal_data (call_frame_t *frame, xlator_t *this, inode_t *inode)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        int_lock->transaction_lk_type = AFR_SELFHEAL_LK;
-        int_lock->selfheal_lk_type    = AFR_DATA_SELF_HEAL_LK;
+	afr_private_t *priv = NULL;
+	unsigned char *locked_on = NULL;
+	int ret = 0;
+	fd_t *fd = NULL;
 
-        afr_set_lock_number (frame, this);
+	priv = this->private;
 
-        int_lock->lk_flock.l_start = start;
-        int_lock->lk_flock.l_len   = len;
-        int_lock->lk_flock.l_type  = F_WRLCK;
-        int_lock->lock_cbk         = afr_sh_data_post_nonblocking_inodelk_cbk;
+	fd = afr_selfheal_data_open (this, inode);
+	if (!fd)
+		return -EIO;
 
-        afr_nonblocking_inodelk (frame, this);
+	locked_on = alloca0 (priv->child_count);
 
-        return 0;
-}
-
-int
-afr_post_sh_big_lock_success (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t     *local     = NULL;
-        afr_self_heal_t *sh        = NULL;
-
-        local = frame->local;
-        sh = &local->self_heal;
-
-        GF_ASSERT (sh->old_loop_frame);
-        sh_loop_finish (sh->old_loop_frame, this);
-        sh->old_loop_frame = NULL;
-        sh->data_lock_held = _gf_true;
-        sh->sync_done = _gf_true;
-        afr_sh_data_fxattrop (frame, this);
-        return 0;
-}
-
-int
-afr_post_sh_big_lock_failure (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t     *local     = NULL;
-        afr_self_heal_t *sh        = NULL;
+	ret = afr_selfheal_tryinodelk (frame, this, inode, priv->sh_domain, 0, 0,
+				       locked_on);
+	{
+		if (ret < 2) {
+			/* Either less than two subvols available, or another
+			   selfheal (from another server) is in progress. Skip
+			   for now in any case there isn't anything to do.
+			*/
+			ret = -ENOTCONN;
+			goto unlock;
+		}
 
-        local = frame->local;
-        sh = &local->self_heal;
+		ret = __afr_selfheal_data (frame, this, fd, locked_on);
+	}
+unlock:
+	afr_selfheal_uninodelk (frame, this, inode, priv->sh_domain, 0, 0, locked_on);
 
-        GF_ASSERT (sh->old_loop_frame);
-        sh_loop_finish (sh->old_loop_frame, this);
-        sh->old_loop_frame = NULL;
-        afr_sh_set_timestamps (frame, this);
-        return 0;
-}
-
-
-int
-afr_sh_data_lock (call_frame_t *frame, xlator_t *this,
-                  off_t start, off_t len,
-                  afr_lock_cbk_t success_handler,
-                  afr_lock_cbk_t failure_handler)
-{
-        afr_local_t *   local = NULL;
-        afr_self_heal_t * sh  = NULL;
-
-        local = frame->local;
-        sh    = &local->self_heal;
-
-        sh->data_lock_success_handler = success_handler;
-        sh->data_lock_failure_handler = failure_handler;
-        return afr_sh_data_lock_rec (frame, this, start, len);
-}
-
-int
-afr_sh_data_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
-{
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        int              call_count = 0;
-        int              child_index = 0;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        child_index = (long) cookie;
-
-        /* TODO: some of the open's might fail.
-           In that case, modify cleanup fn to send flush on those
-           fd's which are already open */
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "open of %s failed on child %s (%s)",
-                                local->loc.path,
-                                priv->children[child_index]->name,
-                                strerror (op_errno));
-                        sh->op_failed = 1;
-                } else {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "open of %s succeeded on child %s",
-                                local->loc.path,
-                                priv->children[child_index]->name);
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                if (sh->op_failed) {
-                        afr_sh_data_fail (frame, this);
-                        return 0;
-                }
-
-                gf_log (this->name, GF_LOG_TRACE,
-                        "fd for %s opened, commencing sync",
-                        local->loc.path);
-
-                afr_sh_data_lock (frame, this, 0, 0,
-                                  afr_sh_data_big_lock_success,
-                                  afr_sh_data_fail);
-        }
-
-        return 0;
-}
-
-
-int
-afr_sh_data_open (call_frame_t *frame, xlator_t *this)
-{
-        int i = 0;
-        int call_count = 0;
-        fd_t *fd = NULL;
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
-        afr_self_heal_t *sh = NULL;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        call_count = afr_up_children_count (local->child_up, priv->child_count);
-        local->call_count = call_count;
-
-        fd = fd_create (local->loc.inode, frame->root->pid);
-        sh->healing_fd = fd;
-
-        /* open sinks */
-        for (i = 0; i < priv->child_count; i++) {
-                if(!local->child_up[i])
-                        continue;
-
-                STACK_WIND_COOKIE (frame, afr_sh_data_open_cbk,
-                                   (void *) (long) i,
-                                   priv->children[i],
-                                   priv->children[i]->fops->open,
-                                   &local->loc,
-                                   O_RDWR|O_LARGEFILE, fd, NULL);
-
-                if (!--call_count)
-                        break;
-        }
-
-        return 0;
-}
-
-
-int
-afr_self_heal_data (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t   *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t *priv = this->private;
-
-        local = frame->local;
-        sh = &local->self_heal;
-
-        /* Self-heal completion cbk changes inode split-brain status based on
-         * govinda_gOvinda, mdata_spb, data_spb values. Initialize data_spb
-         * with current split-brain status. If for some reason self-heal
-         * fails(locking phase etc), it makes sure we retain the split-brain
-         * status before this self-heal started.
-         */
-        sh->data_spb = afr_is_split_brain (this, sh->inode);
-        if (sh->do_data_self_heal &&
-            afr_data_self_heal_enabled (priv->data_self_heal)) {
-                afr_sh_data_open (frame, this);
-        } else {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "not doing data self heal on %s",
-                        local->loc.path);
-                afr_sh_data_done (frame, this);
-        }
+	if (fd)
+		fd_unref (fd);
 
-        return 0;
+	return ret;
 }
diff --git a/xlators/cluster/afr/src/afr-self-heal-entry.c b/xlators/cluster/afr/src/afr-self-heal-entry.c
index 4db47c257..9605d69f4 100644
--- a/xlators/cluster/afr/src/afr-self-heal-entry.c
+++ b/xlators/cluster/afr/src/afr-self-heal-entry.c
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -8,2407 +8,624 @@
   cases as published by the Free Software Foundation.
 */
 
-#include <libgen.h>
-#include <unistd.h>
-#include <fnmatch.h>
-#include <sys/time.h>
-#include <stdlib.h>
-#include <signal.h>
 
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
 #endif
 
-#include "glusterfs.h"
-#include "inode.h"
 #include "afr.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "stack.h"
-#include "list.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include "compat-errno.h"
-#include "compat.h"
+#include "afr-self-heal.h"
 #include "byte-order.h"
-
 #include "afr-transaction.h"
-#include "afr-self-heal.h"
-#include "afr-self-heal-common.h"
-
-#define AFR_INIT_SH_FRAME_VALS(_frame, _local, _sh, _sh_frame, _sh_local, _sh_sh)\
-        do {\
-                _local = _frame->local;\
-                _sh = &_local->self_heal;\
-                _sh_frame = _sh->sh_frame;\
-                _sh_local = _sh_frame->local;\
-                _sh_sh    = &_sh_local->self_heal;\
-        } while (0);
-
-int
-afr_sh_entry_impunge_create_file (call_frame_t *impunge_frame, xlator_t *this,
-                                  int child_index);
-int
-afr_sh_entry_done (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-
-        local = frame->local;
-        sh = &local->self_heal;
-
-        sh->completion_cbk (frame, this);
-
-        return 0;
-}
-
-
-int
-afr_sh_entry_unlock (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t         *local    = NULL;
-        afr_internal_lock_t *int_lock = NULL;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        int_lock->lock_cbk = afr_sh_entry_done;
-        afr_unlock (frame, this);
-
-        return 0;
-}
-
-
-int
-afr_sh_entry_finish (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t   *local = NULL;
-
-        local = frame->local;
-
-        gf_log (this->name, GF_LOG_TRACE,
-                "finishing entry selfheal of %s", local->loc.path);
-
-        afr_sh_entry_unlock (frame, this);
-
-        return 0;
-}
-
-
-int
-afr_sh_entry_erase_pending_cbk (call_frame_t *frame, void *cookie,
-                                xlator_t *this, int32_t op_ret,
-                                int32_t op_errno, dict_t *xattr, dict_t *xdata)
-{
-        long                 i          = 0;
-        int                  call_count = 0;
-        afr_local_t         *local      = NULL;
-        afr_self_heal_t     *sh         = NULL;
-        afr_local_t         *orig_local = NULL;
-        call_frame_t        *orig_frame = NULL;
-        afr_private_t       *priv       = NULL;
-        int32_t             read_child  = -1;
-
-        local = frame->local;
-        priv  = this->private;
-        sh = &local->self_heal;
-        i = (long)cookie;
-
-
-        afr_children_add_child (sh->fresh_children, i, priv->child_count);
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%s: failed to erase pending xattrs on %s (%s)",
-                        local->loc.path, priv->children[i]->name,
-                        strerror (op_errno));
-        }
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                if (sh->source == -1) {
-                        //this happens if the forced merge option is set
-                        read_child = sh->fresh_children[0];
-                } else {
-                        read_child = sh->source;
-                }
-                afr_inode_set_read_ctx (this, sh->inode, read_child,
-                                        sh->fresh_children);
-                orig_frame = sh->orig_frame;
-                orig_local = orig_frame->local;
-
-                if (sh->source != -1) {
-                        orig_local->cont.lookup.buf.ia_nlink = sh->buf[sh->source].ia_nlink;
-                }
-
-                afr_sh_entry_finish (frame, this);
-        }
-
-        return 0;
-}
-
-
-int
-afr_sh_entry_erase_pending (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        int              call_count = 0;
-        int              i = 0;
-        dict_t          **erase_xattr = NULL;
-        int              need_unwind = 0;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        if (sh->entries_skipped) {
-                need_unwind = 1;
-                sh->op_failed = _gf_true;
-                goto out;
-        }
-        afr_sh_pending_to_delta (priv, sh->xattr, sh->delta_matrix, sh->success,
-                                 priv->child_count, AFR_ENTRY_TRANSACTION);
-
-        erase_xattr = GF_CALLOC (sizeof (*erase_xattr), priv->child_count,
-                                 gf_afr_mt_dict_t);
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (sh->xattr[i]) {
-                        call_count++;
-
-                        erase_xattr[i] = get_new_dict();
-                        dict_ref (erase_xattr[i]);
-                }
-        }
-
-        if (call_count == 0)
-                need_unwind = 1;
-
-        afr_sh_delta_to_xattr (priv, sh->delta_matrix, erase_xattr,
-                               priv->child_count, AFR_ENTRY_TRANSACTION);
-
-        local->call_count = call_count;
-        for (i = 0; i < priv->child_count; i++) {
-                if (!erase_xattr[i])
-                        continue;
-
-                gf_log (this->name, GF_LOG_TRACE,
-                        "erasing pending flags from %s on %s",
-                        local->loc.path, priv->children[i]->name);
-
-                STACK_WIND_COOKIE (frame, afr_sh_entry_erase_pending_cbk,
-                                   (void *) (long) i,
-                                   priv->children[i],
-                                   priv->children[i]->fops->xattrop,
-                                   &local->loc,
-                                   GF_XATTROP_ADD_ARRAY, erase_xattr[i],
-                                   NULL);
-                if (!--call_count)
-                        break;
-        }
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (erase_xattr[i]) {
-                        dict_unref (erase_xattr[i]);
-                }
-        }
-        GF_FREE (erase_xattr);
-
-out:
-        if (need_unwind)
-                afr_sh_entry_finish (frame, this);
-
-        return 0;
-}
-
 
 
 static int
-next_active_source (call_frame_t *frame, xlator_t *this,
-                    int current_active_source)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *local  = NULL;
-        afr_self_heal_t *sh  = NULL;
-        int              source = -1;
-        int              next_active_source = -1;
-        int              i = 0;
-
-        priv = this->private;
-        local = frame->local;
-        sh = &local->self_heal;
-
-        source = sh->source;
-
-        if (source != -1) {
-                if (current_active_source != source)
-                        next_active_source = source;
-                goto out;
-        }
-
-        /*
-          the next active sink becomes the source for the
-          'conservative decision' of merging all entries
-        */
-
-        for (i = 0; i < priv->child_count; i++) {
-                if ((sh->sources[i] == 0)
-                    && (local->child_up[i] == 1)
-                    && (i > current_active_source)) {
-
-                        next_active_source = i;
-                        break;
-                }
-        }
-out:
-        return next_active_source;
-}
-
-
-
-static int
-next_active_sink (call_frame_t *frame, xlator_t *this,
-                  int current_active_sink)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *local  = NULL;
-        afr_self_heal_t *sh  = NULL;
-        int              next_active_sink = -1;
-        int              i = 0;
-
-        priv = this->private;
-        local = frame->local;
-        sh = &local->self_heal;
-
-        /*
-          the next active sink becomes the source for the
-          'conservative decision' of merging all entries
-        */
-
-        for (i = 0; i < priv->child_count; i++) {
-                if ((sh->sources[i] == 0)
-                    && (local->child_up[i] == 1)
-                    && (i > current_active_sink)) {
-
-                        next_active_sink = i;
-                        break;
-                }
-        }
-
-        return next_active_sink;
-}
-
-int
-afr_sh_entry_impunge_all (call_frame_t *frame, xlator_t *this);
-
-int
-afr_sh_entry_impunge_subvol (call_frame_t *frame, xlator_t *this);
+afr_selfheal_entry_delete (call_frame_t *frame, xlator_t *this, inode_t *dir,
+			   const char *name, inode_t *inode, int child,
+			   struct afr_reply *replies)
+{
+	afr_private_t *priv = NULL;
+	xlator_t *subvol = NULL;
+	int ret = 0;
+	loc_t loc = {0, };
+	char g[64];
+
+	priv = this->private;
+
+	subvol = priv->children[child];
+
+	loc.parent = inode_ref (dir);
+	uuid_copy (loc.pargfid, dir->gfid);
+	loc.name = name;
+	loc.inode = inode_ref (inode);
+
+	if (replies[child].valid && replies[child].op_ret == 0) {
+		switch (replies[child].poststat.ia_type) {
+		case IA_IFDIR:
+			gf_log (this->name, GF_LOG_WARNING,
+				"expunging dir %s/%s (%s) on %s",
+				uuid_utoa (dir->gfid), name,
+				uuid_utoa_r (replies[child].poststat.ia_gfid, g),
+				subvol->name);
+			ret = syncop_rmdir (subvol, &loc, 1);
+			break;
+		default:
+			gf_log (this->name, GF_LOG_WARNING,
+				"expunging file %s/%s (%s) on %s",
+				uuid_utoa (dir->gfid), name,
+				uuid_utoa_r (replies[child].poststat.ia_gfid, g),
+				subvol->name);
+			ret = syncop_unlink (subvol, &loc);
+			break;
+		}
+	}
+
+	loc_wipe (&loc);
+
+	return ret;
+}
+
+
+int
+afr_selfheal_recreate_entry (call_frame_t *frame, xlator_t *this, int dst,
+			     int source, inode_t *dir, const char *name,
+			     inode_t *inode, struct afr_reply *replies)
+{
+	int ret = 0;
+	loc_t loc = {0,};
+	loc_t srcloc = {0,};
+	afr_private_t *priv = NULL;
+	dict_t *xdata = NULL;
+	struct iatt *iatt = NULL;
+	char *linkname = NULL;
+	mode_t mode = 0;
+	struct iatt newent = {0,};
+
+	priv = this->private;
+
+	xdata = dict_new();
+	if (!xdata)
+		return -ENOMEM;
+
+	loc.parent = inode_ref (dir);
+	uuid_copy (loc.pargfid, dir->gfid);
+	loc.name = name;
+	loc.inode = inode_ref (inode);
+
+	ret = afr_selfheal_entry_delete (frame, this, dir, name, inode, dst,
+					 replies);
+	if (ret)
+		goto out;
+
+	ret = dict_set_static_bin (xdata, "gfid-req",
+				   replies[source].poststat.ia_gfid, 16);
+	if (ret)
+		goto out;
+
+	iatt = &replies[source].poststat;
+
+	srcloc.inode = inode_ref (inode);
+	uuid_copy (srcloc.gfid, iatt->ia_gfid);
+
+	mode = st_mode_from_ia (iatt->ia_prot, iatt->ia_type);
+
+	switch (iatt->ia_type) {
+	case IA_IFDIR:
+		ret = syncop_mkdir (priv->children[dst], &loc, mode, xdata, 0);
+		break;
+	case IA_IFLNK:
+		ret = syncop_lookup (priv->children[dst], &srcloc, 0, 0, 0, 0);
+		if (ret == 0) {
+			ret = syncop_link (priv->children[dst], &srcloc, &loc);
+		} else {
+			ret = syncop_readlink (priv->children[source], &srcloc,
+					       &linkname, 4096);
+			if (ret <= 0)
+				goto out;
+			ret = syncop_symlink (priv->children[dst], &loc, linkname,
+					      xdata, NULL);
+		}
+		break;
+	default:
+		ret = dict_set_int32 (xdata, GLUSTERFS_INTERNAL_FOP_KEY, 1);
+		if (ret)
+			goto out;
+		ret = syncop_mknod (priv->children[dst], &loc, mode,
+				    iatt->ia_rdev, xdata, &newent);
+		if (ret == 0 && iatt->ia_size && !newent.ia_size) {
+			/* New entry created. Mark @dst pending on all sources */
+			ret = 1;
+		}
+		break;
+	}
 
-int
-afr_sh_entry_expunge_all (call_frame_t *frame, xlator_t *this);
-
-int
-afr_sh_entry_expunge_subvol (call_frame_t *frame, xlator_t *this,
-                             int active_src);
-
-int
-afr_sh_entry_expunge_entry_done (call_frame_t *frame, xlator_t *this,
-                                 int active_src, int32_t op_ret,
-                                 int32_t op_errno)
-{
-        int              call_count = 0;
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0)
-                afr_sh_entry_expunge_subvol (frame, this, active_src);
-
-        return 0;
-}
-
-int
-afr_sh_entry_expunge_parent_setattr_cbk (call_frame_t *expunge_frame,
-                                         void *cookie, xlator_t *this,
-                                         int32_t op_ret, int32_t op_errno,
-                                         struct iatt *preop, struct iatt *postop,
-                                         dict_t *xdata)
-{
-        afr_private_t   *priv          = NULL;
-        afr_local_t     *expunge_local = NULL;
-        afr_self_heal_t *expunge_sh    = NULL;
-        call_frame_t    *frame         = NULL;
-        int              active_src    = (long) cookie;
-        afr_self_heal_t *sh            = NULL;
-        afr_local_t     *local         = NULL;
-
-        priv          = this->private;
-        expunge_local = expunge_frame->local;
-        expunge_sh    = &expunge_local->self_heal;
-        frame         = expunge_sh->sh_frame;
-        local         = frame->local;
-        sh            = &local->self_heal;
-
-        if (op_ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "setattr on parent directory of %s on subvolume %s failed: %s",
-                        expunge_local->loc.path,
-                        priv->children[active_src]->name, strerror (op_errno));
-        }
-
-        AFR_STACK_DESTROY (expunge_frame);
-        sh->expunge_done (frame, this, active_src, op_ret, op_errno);
-
-        return 0;
-}
-
-
-int
-afr_sh_entry_expunge_remove_cbk (call_frame_t *expunge_frame, void *cookie,
-                                 xlator_t *this,
-                                 int32_t op_ret, int32_t op_errno,
-                                 struct iatt *preparent,
-                                 struct iatt *postparent, dict_t *xdata)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *expunge_local = NULL;
-        afr_self_heal_t *expunge_sh = NULL;
-        int              active_src = 0;
-        int32_t          valid = 0;
-
-        priv = this->private;
-        expunge_local = expunge_frame->local;
-        expunge_sh = &expunge_local->self_heal;
-
-        active_src = (long) cookie;
-
-        if (op_ret == 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "removed %s on %s",
-                        expunge_local->loc.path,
-                        priv->children[active_src]->name);
-        } else {
-                gf_log (this->name, GF_LOG_INFO,
-                        "removing %s on %s failed (%s)",
-                        expunge_local->loc.path,
-                        priv->children[active_src]->name,
-                        strerror (op_errno));
-        }
-
-        valid = GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME;
-
-        STACK_WIND_COOKIE (expunge_frame, afr_sh_entry_expunge_parent_setattr_cbk,
-                           (void *) (long) active_src,
-                           priv->children[active_src],
-                           priv->children[active_src]->fops->setattr,
-                           &expunge_sh->parent_loc,
-                           &expunge_sh->parentbuf,
-                           valid, NULL);
-
-        return 0;
-}
-
-
-int
-afr_sh_entry_expunge_unlink (call_frame_t *expunge_frame, xlator_t *this,
-                             int active_src)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *expunge_local = NULL;
-
-        priv          = this->private;
-        expunge_local = expunge_frame->local;
-
-        gf_log (this->name, GF_LOG_TRACE,
-                "expunging file %s on %s",
-                expunge_local->loc.path, priv->children[active_src]->name);
-
-        STACK_WIND_COOKIE (expunge_frame, afr_sh_entry_expunge_remove_cbk,
-                           (void *) (long) active_src,
-                           priv->children[active_src],
-                           priv->children[active_src]->fops->unlink,
-                           &expunge_local->loc, 0, NULL);
-
-        return 0;
-}
-
-
-
-int
-afr_sh_entry_expunge_rmdir (call_frame_t *expunge_frame, xlator_t *this,
-                            int active_src)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *expunge_local = NULL;
-
-        priv          = this->private;
-        expunge_local = expunge_frame->local;
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "expunging directory %s on %s",
-                expunge_local->loc.path, priv->children[active_src]->name);
-
-        STACK_WIND_COOKIE (expunge_frame, afr_sh_entry_expunge_remove_cbk,
-                           (void *) (long) active_src,
-                           priv->children[active_src],
-                           priv->children[active_src]->fops->rmdir,
-                           &expunge_local->loc, 1, NULL);
-
-        return 0;
-}
-
-
-int
-afr_sh_entry_expunge_remove (call_frame_t *expunge_frame, xlator_t *this,
-                             int active_src, struct iatt *buf,
-                             struct iatt *parentbuf)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *expunge_local = NULL;
-        afr_self_heal_t *expunge_sh = NULL;
-        call_frame_t    *frame = NULL;
-        int              type = 0;
-        afr_self_heal_t *sh            = NULL;
-        afr_local_t     *local         = NULL;
-        loc_t           *loc           = NULL;
-
-        priv = this->private;
-        expunge_local = expunge_frame->local;
-        expunge_sh = &expunge_local->self_heal;
-        frame = expunge_sh->sh_frame;
-        local         = frame->local;
-        sh            = &local->self_heal;
-        loc           = &expunge_local->loc;
-
-        type = buf->ia_type;
-        if (loc->parent && uuid_is_null (loc->parent->gfid))
-                uuid_copy (loc->pargfid, parentbuf->ia_gfid);
-
-        switch (type) {
-        case IA_IFSOCK:
-        case IA_IFREG:
-        case IA_IFBLK:
-        case IA_IFCHR:
-        case IA_IFIFO:
-        case IA_IFLNK:
-                afr_sh_entry_expunge_unlink (expunge_frame, this, active_src);
-                break;
-        case IA_IFDIR:
-                afr_sh_entry_expunge_rmdir (expunge_frame, this, active_src);
-                break;
-        default:
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s has unknown file type on %s: 0%o",
-                        expunge_local->loc.path,
-                        priv->children[active_src]->name, type);
-                goto out;
-                break;
-        }
-
-        return 0;
-out:
-        AFR_STACK_DESTROY (expunge_frame);
-        sh->expunge_done (frame, this, active_src, -1, EINVAL);
-
-        return 0;
-}
-
-
-int
-afr_sh_entry_expunge_lookup_cbk (call_frame_t *expunge_frame, void *cookie,
-                                 xlator_t *this,
-                                 int32_t op_ret, int32_t op_errno,
-                                 inode_t *inode, struct iatt *buf, dict_t *x,
-                                 struct iatt *postparent)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *expunge_local = NULL;
-        afr_self_heal_t *expunge_sh = NULL;
-        call_frame_t    *frame = NULL;
-        int              active_src = 0;
-        afr_self_heal_t *sh            = NULL;
-        afr_local_t     *local         = NULL;
-
-        priv = this->private;
-        expunge_local = expunge_frame->local;
-        expunge_sh = &expunge_local->self_heal;
-        frame = expunge_sh->sh_frame;
-        active_src = (long) cookie;
-        local         = frame->local;
-        sh            = &local->self_heal;
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "lookup of %s on %s failed (%s)",
-                        expunge_local->loc.path,
-                        priv->children[active_src]->name,
-                        strerror (op_errno));
-                goto out;
-        }
-
-        afr_sh_entry_expunge_remove (expunge_frame, this, active_src, buf,
-                                     postparent);
-
-        return 0;
 out:
-        AFR_STACK_DESTROY (expunge_frame);
-        sh->expunge_done (frame, this, active_src, op_ret, op_errno);
-
-        return 0;
+	if (xdata)
+		dict_unref (xdata);
+	loc_wipe (&loc);
+	loc_wipe (&srcloc);
+	return ret;
 }
 
 
-int
-afr_sh_entry_expunge_purge (call_frame_t *expunge_frame, xlator_t *this,
-                            int active_src)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *expunge_local = NULL;
-
-        priv = this->private;
-        expunge_local = expunge_frame->local;
-
-        gf_log (this->name, GF_LOG_TRACE,
-                "looking up %s on %s",
-                expunge_local->loc.path, priv->children[active_src]->name);
-
-        STACK_WIND_COOKIE (expunge_frame, afr_sh_entry_expunge_lookup_cbk,
-                           (void *) (long) active_src,
-                           priv->children[active_src],
-                           priv->children[active_src]->fops->lookup,
-                           &expunge_local->loc, NULL);
-
-        return 0;
-}
-
-int
-afr_sh_entry_expunge_entry_cbk (call_frame_t *expunge_frame, void *cookie,
-                                xlator_t *this,
-                                int32_t op_ret, int32_t op_errno,
-                                inode_t *inode, struct iatt *buf, dict_t *x,
-                                struct iatt *postparent)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *expunge_local = NULL;
-        afr_self_heal_t *expunge_sh = NULL;
-        int              source = 0;
-        call_frame_t    *frame = NULL;
-        int              active_src = 0;
-        int              need_expunge = 0;
-        afr_self_heal_t *sh            = NULL;
-        afr_local_t     *local         = NULL;
-
-        priv = this->private;
-        expunge_local = expunge_frame->local;
-        expunge_sh = &expunge_local->self_heal;
-        frame = expunge_sh->sh_frame;
-        active_src = expunge_sh->active_source;
-        source = (long) cookie;
-        local         = frame->local;
-        sh            = &local->self_heal;
-
-        if (op_ret == -1 && op_errno == ENOENT)
-                need_expunge = 1;
-        else if (op_ret == -1)
-                goto out;
-
-        if (!uuid_is_null (expunge_sh->entrybuf.ia_gfid) &&
-            !uuid_is_null (buf->ia_gfid) &&
-            (uuid_compare (expunge_sh->entrybuf.ia_gfid, buf->ia_gfid) != 0)) {
-                char uuidbuf1[64];
-                char uuidbuf2[64];
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "entry %s found on %s with mismatching gfid (%s/%s)",
-                        expunge_local->loc.path,
-                        priv->children[source]->name,
-                        uuid_utoa_r (expunge_sh->entrybuf.ia_gfid, uuidbuf1),
-                        uuid_utoa_r (buf->ia_gfid, uuidbuf2));
-                need_expunge = 1;
-        }
-
-        if (need_expunge) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "missing entry %s on %s",
-                        expunge_local->loc.path,
-                        priv->children[source]->name);
-
-                if (postparent)
-                        expunge_sh->parentbuf = *postparent;
-
-                afr_sh_entry_expunge_purge (expunge_frame, this, active_src);
-
-                return 0;
-        }
-
-out:
-        if (op_ret == 0) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "%s exists under %s",
-                        expunge_local->loc.path,
-                        priv->children[source]->name);
-        } else {
-                gf_log (this->name, GF_LOG_INFO,
-                        "looking up %s under %s failed (%s)",
-                        expunge_local->loc.path,
-                        priv->children[source]->name,
-                        strerror (op_errno));
-        }
-
-        AFR_STACK_DESTROY (expunge_frame);
-        sh->expunge_done (frame, this, active_src, op_ret, op_errno);
-
-        return 0;
-}
-
-
-int
-afr_sh_entry_expunge_entry (call_frame_t *frame, xlator_t *this,
-                            gf_dirent_t *entry)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *local  = NULL;
-        afr_self_heal_t *sh  = NULL;
-        int              ret = -1;
-        call_frame_t    *expunge_frame = NULL;
-        afr_local_t     *expunge_local = NULL;
-        afr_self_heal_t *expunge_sh = NULL;
-        int              active_src = 0;
-        int              source = 0;
-        int              op_errno = 0;
-        char            *name = NULL;
-        int             op_ret = -1;
-
-        priv = this->private;
-        local = frame->local;
-        sh = &local->self_heal;
-
-        active_src = sh->active_source;
-        source = sh->source;
-        sh->expunge_done = afr_sh_entry_expunge_entry_done;
-
-        name = entry->d_name;
-
-        if ((strcmp (name, ".") == 0)
-            || (strcmp (name, "..") == 0)
-            || ((strcmp (local->loc.path, "/") == 0)
-                && (strcmp (name, GF_REPLICATE_TRASH_DIR) == 0))) {
-
-                gf_log (this->name, GF_LOG_TRACE,
-                        "skipping inspection of %s under %s",
-                        name, local->loc.path);
-                op_ret = 0;
-                goto out;
-        }
-
-        gf_log (this->name, GF_LOG_TRACE,
-                "inspecting existence of %s under %s",
-                name, local->loc.path);
-
-        expunge_frame = copy_frame (frame);
-        if (!expunge_frame) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        AFR_LOCAL_ALLOC_OR_GOTO (expunge_local, out);
-
-        expunge_frame->local = expunge_local;
-        expunge_sh = &expunge_local->self_heal;
-        expunge_sh->sh_frame = frame;
-        expunge_sh->active_source = active_src;
-        expunge_sh->entrybuf = entry->d_stat;
-        loc_copy (&expunge_sh->parent_loc, &local->loc);
-
-        ret = afr_build_child_loc (this, &expunge_local->loc, &local->loc,
-                                   name);
-        if (ret != 0) {
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        gf_log (this->name, GF_LOG_TRACE,
-                "looking up %s on %s", expunge_local->loc.path,
-                priv->children[source]->name);
-
-        STACK_WIND_COOKIE (expunge_frame,
-                           afr_sh_entry_expunge_entry_cbk,
-                           (void *) (long) source,
-                           priv->children[source],
-                           priv->children[source]->fops->lookup,
-                           &expunge_local->loc, NULL);
-
-        ret = 0;
-out:
-        if (ret == -1)
-                sh->expunge_done (frame, this, active_src, op_ret, op_errno);
-
-        return 0;
-}
-
-
-int
-afr_sh_entry_expunge_readdir_cbk (call_frame_t *frame, void *cookie,
-                                  xlator_t *this,
-                                  int32_t op_ret, int32_t op_errno,
-                                  gf_dirent_t *entries, dict_t *xdata)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *local  = NULL;
-        afr_self_heal_t *sh  = NULL;
-        gf_dirent_t     *entry = NULL;
-        off_t            last_offset = 0;
-        int              active_src = 0;
-        int              entry_count = 0;
-
-        priv = this->private;
-        local = frame->local;
-        sh = &local->self_heal;
-
-        active_src = sh->active_source;
-
-        if (op_ret <= 0) {
-                if (op_ret < 0) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "readdir of %s on subvolume %s failed (%s)",
-                                local->loc.path,
-                                priv->children[active_src]->name,
-                                strerror (op_errno));
-                } else {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "readdir of %s on subvolume %s complete",
-                                local->loc.path,
-                                priv->children[active_src]->name);
-                }
-
-                afr_sh_entry_expunge_all (frame, this);
-                return 0;
-        }
-
-        list_for_each_entry (entry, &entries->list, list) {
-                last_offset = entry->d_off;
-                entry_count++;
-        }
-
-        gf_log (this->name, GF_LOG_TRACE,
-                "readdir'ed %d entries from %s",
-                entry_count, priv->children[active_src]->name);
-
-        sh->offset = last_offset;
-        local->call_count = entry_count;
-
-        list_for_each_entry (entry, &entries->list, list) {
-                afr_sh_entry_expunge_entry (frame, this, entry);
-        }
-
-        return 0;
-}
-
-int
-afr_sh_entry_expunge_subvol (call_frame_t *frame, xlator_t *this,
-                             int active_src)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *local  = NULL;
-        afr_self_heal_t *sh  = NULL;
-
-        priv = this->private;
-        local = frame->local;
-        sh = &local->self_heal;
-
-        STACK_WIND (frame, afr_sh_entry_expunge_readdir_cbk,
-                    priv->children[active_src],
-                    priv->children[active_src]->fops->readdirp,
-                    sh->healing_fd, sh->block_size, sh->offset, NULL);
-
-        return 0;
-}
-
-
-int
-afr_sh_entry_expunge_all (call_frame_t *frame, xlator_t *this)
+static int
+afr_selfheal_newentry_mark (call_frame_t *frame, xlator_t *this, inode_t *inode,
+			    int source, struct afr_reply *replies,
+			    unsigned char *sources, unsigned char *newentry)
 {
-        afr_private_t   *priv = NULL;
-        afr_local_t     *local  = NULL;
-        afr_self_heal_t *sh  = NULL;
-        int              active_src = -1;
-
-        priv = this->private;
-        local = frame->local;
-        sh = &local->self_heal;
+	int ret = 0;
+	int i = 0;
+	afr_private_t *priv = NULL;
+	dict_t *xattr = NULL;
+	int **changelog = NULL;
+	int idx = 0;
 
-        sh->offset = 0;
+	priv = this->private;
 
-        if (sh->source == -1) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "no active sources for %s to expunge entries",
-                        local->loc.path);
-                goto out;
-        }
+	idx = afr_index_for_transaction_type (AFR_DATA_TRANSACTION);
 
-        active_src = next_active_sink (frame, this, sh->active_source);
-        sh->active_source = active_src;
+	uuid_copy (inode->gfid, replies[source].poststat.ia_gfid);
 
-        if (sh->op_failed) {
-                goto out;
-        }
+	changelog = afr_matrix_create (priv->child_count, AFR_NUM_CHANGE_LOGS);
 
-        if (active_src == -1) {
-                /* completed creating missing files on all subvolumes */
-                goto out;
-        }
+	xattr = dict_new();
+	if (!xattr)
+		return -ENOMEM;
 
-        gf_log (this->name, GF_LOG_TRACE,
-                "expunging entries of %s on %s to other sinks",
-                local->loc.path, priv->children[active_src]->name);
+	for (i = 0; i < priv->child_count; i++) {
+		if (!newentry[i])
+			continue;
+		changelog[i][idx] = hton32(1);
+	}
 
-        afr_sh_entry_expunge_subvol (frame, this, active_src);
-
-        return 0;
-out:
-        afr_sh_entry_impunge_all (frame, this);
-        return 0;
-
-}
+	afr_set_pending_dict (priv, xattr, changelog);
 
+	for (i = 0; i < priv->child_count; i++) {
+		if (!sources[i])
+			continue;
+		afr_selfheal_post_op (frame, this, inode, i, xattr);
+	}
 
-int
-afr_sh_entry_impunge_entry_done (call_frame_t *frame, xlator_t *this,
-                                 int32_t op_ret, int32_t op_errno)
-{
-        int              call_count = 0;
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        if (op_ret < 0)
-                sh->entries_skipped = _gf_true;
-        call_count = afr_frame_return (frame);
-        if (call_count == 0)
-                afr_sh_entry_impunge_subvol (frame, this);
-
-        return 0;
+	dict_unref (xattr);
+	return ret;
 }
 
-void
-afr_sh_entry_call_impunge_done (call_frame_t *impunge_frame, xlator_t *this,
-                                int32_t op_ret, int32_t op_errno)
-{
-        afr_local_t     *impunge_local = NULL;
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_self_heal_t *impunge_sh = NULL;
-        call_frame_t    *frame = NULL;
 
-        AFR_INIT_SH_FRAME_VALS (impunge_frame, impunge_local, impunge_sh,
-                                frame, local, sh);
-
-        AFR_STACK_DESTROY (impunge_frame);
-        sh->impunge_done (frame, this, op_ret, op_errno);
+static int
+__afr_selfheal_heal_dirent (call_frame_t *frame, xlator_t *this, fd_t *fd,
+			    char *name, inode_t *inode, int source,
+			    unsigned char *sources, unsigned char *healed_sinks,
+			    unsigned char *locked_on, struct afr_reply *replies)
+{
+	int ret = 0;
+	afr_private_t *priv = NULL;
+	int i = 0;
+	unsigned char *newentry = NULL;
+
+	priv = this->private;
+	newentry = alloca0 (priv->child_count);
+
+	if (!replies[source].valid)
+		return -EIO;
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!healed_sinks[i])
+			continue;
+		if (replies[source].op_ret == -1 &&
+		    replies[source].op_errno == ENOENT) {
+			ret = afr_selfheal_entry_delete (frame, this, fd->inode,
+							 name, inode, i, replies);
+		} else {
+			if (!uuid_compare (replies[i].poststat.ia_gfid,
+					   replies[source].poststat.ia_gfid))
+				continue;
+
+			ret = afr_selfheal_recreate_entry (frame, this, i, source,
+							   fd->inode, name, inode,
+							   replies);
+			if (ret > 0) {
+				newentry[i] = 1;
+				ret = 0;
+			}
+		}
+		if (ret < 0)
+			break;
+	}
+
+	if (AFR_COUNT (newentry, priv->child_count))
+		afr_selfheal_newentry_mark (frame, this, inode, source, replies,
+					    sources, newentry);
+	return ret;
 }
 
-int
-afr_sh_entry_impunge_setattr_cbk (call_frame_t *impunge_frame, void *cookie,
-                                  xlator_t *this,
-                                  int32_t op_ret, int32_t op_errno,
-                                  struct iatt *preop, struct iatt *postop,
-                                  dict_t *xdata)
-{
-        int              call_count = 0;
-        afr_private_t   *priv = NULL;
-        afr_local_t     *impunge_local = NULL;
-        int              child_index = 0;
-
-        priv = this->private;
-        impunge_local = impunge_frame->local;
-        child_index = (long) cookie;
-
-        if (op_ret == 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "setattr done for %s on %s",
-                        impunge_local->loc.path,
-                        priv->children[child_index]->name);
-        } else {
-                gf_log (this->name, GF_LOG_INFO,
-                        "setattr (%s) on %s failed (%s)",
-                        impunge_local->loc.path,
-                        priv->children[child_index]->name,
-                        strerror (op_errno));
-        }
-
-        call_count = afr_frame_return (impunge_frame);
-        if (call_count == 0) {
-                afr_sh_entry_call_impunge_done (impunge_frame, this,
-                                                0, op_errno);
-        }
-
-        return 0;
-}
 
-int
-afr_sh_entry_impunge_parent_setattr_cbk (call_frame_t *setattr_frame,
-                                         void *cookie, xlator_t *this,
-                                         int32_t op_ret, int32_t op_errno,
-                                         struct iatt *preop, struct iatt *postop,
-                                         dict_t *xdata)
+static int
+__afr_selfheal_merge_dirent (call_frame_t *frame, xlator_t *this, fd_t *fd,
+			     char *name, inode_t *inode, unsigned char *sources,
+			     unsigned char *healed_sinks, unsigned char *locked_on,
+			     struct afr_reply *replies)
 {
-        int             call_count = 0;
-        afr_local_t     *setattr_local = NULL;
-
-        setattr_local = setattr_frame->local;
-        if (op_ret != 0) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "setattr on parent directory (%s) failed: %s",
-                        setattr_local->loc.path, strerror (op_errno));
-        }
-
-        call_count = afr_frame_return (setattr_frame);
-        if (call_count == 0)
-                AFR_STACK_DESTROY (setattr_frame);
-        return 0;
-}
+	int ret = 0;
+	afr_private_t *priv = NULL;
+	int i = 0;
+	int source = -1;
 
-int
-afr_sh_entry_impunge_setattr (call_frame_t *impunge_frame, xlator_t *this)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *impunge_local = NULL;
-        afr_local_t     *setattr_local = NULL;
-        afr_self_heal_t *impunge_sh = NULL;
-        call_frame_t    *setattr_frame = NULL;
-        int32_t          valid = 0;
-        int32_t          op_errno = 0;
-        int              child_index = 0;
-        int              call_count = 0;
-        int              i = 0;
-
-        priv          = this->private;
-        impunge_local = impunge_frame->local;
-        impunge_sh    = &impunge_local->self_heal;
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "setting ownership of %s on %s to %d/%d",
-                impunge_local->loc.path,
-                priv->children[child_index]->name,
-                impunge_sh->entrybuf.ia_uid,
-                impunge_sh->entrybuf.ia_gid);
-
-        setattr_frame = copy_frame (impunge_frame);
-        if (!setattr_frame) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-        AFR_LOCAL_ALLOC_OR_GOTO (setattr_frame->local, out);
-        setattr_local = setattr_frame->local;
-        call_count = afr_errno_count (NULL, impunge_sh->child_errno,
-                                      priv->child_count, 0);
-        loc_copy (&setattr_local->loc, &impunge_sh->parent_loc);
-        impunge_local->call_count = call_count;
-        setattr_local->call_count = call_count;
-        for (i = 0; i < priv->child_count; i++) {
-                if (impunge_sh->child_errno[i])
-                        continue;
-                valid         = GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME;
-                STACK_WIND_COOKIE (setattr_frame,
-                                   afr_sh_entry_impunge_parent_setattr_cbk,
-                                   (void *) (long) i, priv->children[i],
-                                   priv->children[i]->fops->setattr,
-                                   &setattr_local->loc,
-                                   &impunge_sh->parentbuf, valid, NULL);
-
-                valid = GF_SET_ATTR_UID   | GF_SET_ATTR_GID |
-                        GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME;
-                STACK_WIND_COOKIE (impunge_frame,
-                                   afr_sh_entry_impunge_setattr_cbk,
-                                   (void *) (long) i, priv->children[i],
-                                   priv->children[i]->fops->setattr,
-                                   &impunge_local->loc,
-                                   &impunge_sh->entrybuf, valid, NULL);
-                call_count--;
-        }
-        GF_ASSERT (!call_count);
-        return 0;
-out:
-        if (setattr_frame)
-                AFR_STACK_DESTROY (setattr_frame);
-        afr_sh_entry_call_impunge_done (impunge_frame, this, 0, op_errno);
-        return 0;
-}
+	priv = this->private;
 
-int
-afr_sh_entry_impunge_xattrop_cbk (call_frame_t *impunge_frame, void *cookie,
-                                  xlator_t *this,
-                                  int32_t op_ret, int32_t op_errno,
-                                  dict_t *xattr, dict_t *xdata)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *impunge_local = NULL;
-        int              child_index = 0;
-
-        priv          = this->private;
-        impunge_local = impunge_frame->local;
-
-        child_index = (long) cookie;
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%s: failed to perform xattrop on %s (%s)",
-                        impunge_local->loc.path,
-                        priv->children[child_index]->name,
-                        strerror (op_errno));
-                goto out;
-        }
-
-        afr_sh_entry_impunge_setattr (impunge_frame, this);
-        return 0;
-out:
-        afr_sh_entry_call_impunge_done (impunge_frame, this,
-                                        -1, op_errno);
-        return 0;
-}
+	for (i = 0; i < priv->child_count; i++) {
+		if (replies[i].valid && replies[i].op_ret == 0) {
+			source = i;
+			break;
+		}
+	}
 
-void
-afr_sh_prepare_new_entry_pending_matrix (int32_t **pending,
-                                         int *child_errno,
-                                         struct iatt *buf,
-                                         unsigned int child_count)
-{
-        int              midx             = 0;
-        int              idx              = 0;
-        int              i                = 0;
-
-        midx = afr_index_for_transaction_type (AFR_METADATA_TRANSACTION);
-        if (IA_ISDIR (buf->ia_type))
-                idx = afr_index_for_transaction_type (AFR_ENTRY_TRANSACTION);
-        else if (IA_ISREG (buf->ia_type))
-                idx = afr_index_for_transaction_type (AFR_DATA_TRANSACTION);
-        else
-                idx = -1;
-        for (i = 0; i < child_count; i++) {
-                if (child_errno[i])
-                        continue;
-                pending[i][midx] = hton32 (1);
-                if (idx == -1)
-                        continue;
-                pending[i][idx] = hton32 (1);
-        }
-}
+	if (source == -1) {
+		/* entry got deleted in the mean time? */
+		return 0;
+	}
 
-int
-afr_sh_entry_impunge_perform_xattrop (call_frame_t *impunge_frame,
-                                      xlator_t *this)
-{
-        int              active_src       = 0;
-        dict_t          *xattr            = NULL;
-        afr_private_t   *priv             = NULL;
-        afr_local_t     *impunge_local    = NULL;
-        afr_self_heal_t *impunge_sh       = NULL;
-        int32_t         op_errno          = 0;
-
-        priv = this->private;
-        impunge_local = impunge_frame->local;
-        impunge_sh = &impunge_local->self_heal;
-        active_src = impunge_sh->active_source;
-
-        afr_sh_prepare_new_entry_pending_matrix (impunge_local->pending,
-                                                 impunge_sh->child_errno,
-                                                 &impunge_sh->entrybuf,
-                                                 priv->child_count);
-        xattr = dict_new ();
-        if (!xattr) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        afr_set_pending_dict (priv, xattr, impunge_local->pending);
-
-        STACK_WIND_COOKIE (impunge_frame, afr_sh_entry_impunge_xattrop_cbk,
-                           (void *) (long) active_src,
-                           priv->children[active_src],
-                           priv->children[active_src]->fops->xattrop,
-                           &impunge_local->loc, GF_XATTROP_ADD_ARRAY, xattr, NULL);
-
-        if (xattr)
-                dict_unref (xattr);
-        return 0;
-out:
-        afr_sh_entry_call_impunge_done (impunge_frame, this,
-                                        -1, op_errno);
-        return 0;
-}
+	for (i = 0; i < priv->child_count; i++) {
+		if (i == source || !healed_sinks[i])
+			continue;
 
-int
-afr_sh_entry_impunge_newfile_cbk (call_frame_t *impunge_frame, void *cookie,
-                                  xlator_t *this,
-                                  int32_t op_ret, int32_t op_errno,
-                                  inode_t *inode, struct iatt *stbuf,
-                                  struct iatt *preparent,
-                                  struct iatt *postparent, dict_t *xdata)
-{
-        int              call_count       = 0;
-        afr_private_t   *priv             = NULL;
-        afr_local_t     *impunge_local    = NULL;
-        afr_self_heal_t *impunge_sh       = NULL;
-        int              child_index      = 0;
-
-        priv = this->private;
-        impunge_local = impunge_frame->local;
-        impunge_sh = &impunge_local->self_heal;
-
-        child_index = (long) cookie;
-
-        if (op_ret == -1) {
-                impunge_sh->child_errno[child_index] = op_errno;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "creation of %s on %s failed (%s)",
-                        impunge_local->loc.path,
-                        priv->children[child_index]->name,
-                        strerror (op_errno));
-        } else {
-                impunge_sh->child_errno[child_index] = 0;
-        }
-
-        call_count = afr_frame_return (impunge_frame);
-        if (call_count == 0) {
-                if (!afr_errno_count (NULL, impunge_sh->child_errno,
-                                      priv->child_count, 0)) {
-                        // new_file creation failed every where
-                        afr_sh_entry_call_impunge_done (impunge_frame, this,
-                                                        -1, op_errno);
-                        goto out;
-                }
-                afr_sh_entry_impunge_perform_xattrop (impunge_frame, this);
-        }
-out:
-        return 0;
-}
+		if (replies[i].op_errno != ENOENT)
+			continue;
 
-int
-afr_sh_entry_impunge_hardlink_cbk (call_frame_t *impunge_frame, void *cookie,
-                                   xlator_t *this, int32_t op_ret,
-                                   int32_t op_errno, inode_t *inode,
-                                   struct iatt *buf, struct iatt *preparent,
-                                   struct iatt *postparent, dict_t *xdata)
-{
-        int              call_count        = 0;
-        afr_local_t     *impunge_local = NULL;
-        afr_self_heal_t *impunge_sh  = NULL;
-
-        impunge_local = impunge_frame->local;
-        impunge_sh = &impunge_local->self_heal;
-
-        if (IA_IFLNK == impunge_sh->entrybuf.ia_type) {
-                //For symlinks impunge is attempted un-conditionally
-                //So the file can already exist.
-                if ((op_ret < 0) && (op_errno == EEXIST))
-                        op_ret = 0;
-        }
-
-        call_count = afr_frame_return (impunge_frame);
-        if (call_count == 0)
-                afr_sh_entry_call_impunge_done (impunge_frame, this,
-                                                op_ret, op_errno);
-
-        return 0;
-}
+		ret = afr_selfheal_recreate_entry (frame, this, i, source,
+						   fd->inode, name, inode,
+						   replies);
+	}
 
-int
-afr_sh_entry_impunge_hardlink (call_frame_t *impunge_frame, xlator_t *this,
-                               int child_index)
-{
-        afr_private_t   *priv          = NULL;
-        afr_local_t     *impunge_local = NULL;
-        afr_self_heal_t *impunge_sh  = NULL;
-        loc_t           *loc           = NULL;
-        struct iatt     *buf           = NULL;
-        loc_t            oldloc        = {0};
-
-        priv = this->private;
-        impunge_local = impunge_frame->local;
-        impunge_sh = &impunge_local->self_heal;
-        loc = &impunge_local->loc;
-        buf = &impunge_sh->entrybuf;
-
-        oldloc.inode = inode_ref (loc->inode);
-        uuid_copy (oldloc.gfid, buf->ia_gfid);
-        gf_log (this->name, GF_LOG_DEBUG, "linking missing file %s on %s",
-                loc->path, priv->children[child_index]->name);
-
-        STACK_WIND_COOKIE (impunge_frame, afr_sh_entry_impunge_hardlink_cbk,
-                           (void *) (long) child_index,
-                           priv->children[child_index],
-                           priv->children[child_index]->fops->link,
-                           &oldloc, loc, NULL);
-        loc_wipe (&oldloc);
-
-        return 0;
+	return ret;
 }
 
-int
-afr_sh_nameless_lookup_cbk (call_frame_t *impunge_frame, void *cookie,
-                            xlator_t *this,
-                            int32_t op_ret, int32_t op_errno, inode_t *inode,
-                            struct iatt *buf, dict_t *xattr,
-                            struct iatt *postparent)
-{
-        if (op_ret < 0) {
-                 afr_sh_entry_impunge_create_file (impunge_frame, this,
-                                                   (long)cookie);
-        } else {
-                afr_sh_entry_impunge_hardlink (impunge_frame, this,
-                                               (long)cookie);
-        }
-        return 0;
-}
 
-int
-afr_sh_entry_impunge_check_hardlink (call_frame_t *impunge_frame,
-                                     xlator_t *this,
-                                     int child_index, struct iatt *stbuf)
+static int
+__afr_selfheal_entry_dirent (call_frame_t *frame, xlator_t *this, fd_t *fd,
+			     char *name, inode_t *inode, int source,
+			     unsigned char *sources, unsigned char *healed_sinks,
+			     unsigned char *locked_on, struct afr_reply *replies)
 {
-        afr_private_t   *priv          = NULL;
-        call_frame_t    *frame             = NULL;
-        afr_local_t     *impunge_local     = NULL;
-        afr_local_t     *local             = NULL;
-        afr_self_heal_t *impunge_sh        = NULL;
-        afr_self_heal_t *sh                = NULL;
-        loc_t           *loc           = NULL;
-        dict_t          *xattr_req     = NULL;
-        loc_t            oldloc        = {0};
-        int              ret           = -1;
-
-        priv = this->private;
-        AFR_INIT_SH_FRAME_VALS (impunge_frame, impunge_local, impunge_sh,
-                                frame, local, sh);
-        loc = &impunge_local->loc;
-
-        xattr_req = dict_new ();
-        if (!xattr_req)
-                goto out;
-        oldloc.inode = inode_ref (loc->inode);
-        uuid_copy (oldloc.gfid, stbuf->ia_gfid);
-
-        STACK_WIND_COOKIE (impunge_frame, afr_sh_nameless_lookup_cbk,
-                           (void *) (long) child_index,
-                           priv->children[child_index],
-                           priv->children[child_index]->fops->lookup,
-                           &oldloc, xattr_req);
-        ret = 0;
-out:
-        if (xattr_req)
-                dict_unref (xattr_req);
-        loc_wipe (&oldloc);
-        if (ret)
-                sh->impunge_done (frame, this, -1, ENOMEM);
-        return 0;
-}
+	int ret = -1;
 
-int
-afr_sh_entry_impunge_mknod (call_frame_t *impunge_frame, xlator_t *this,
-                            int child_index, struct iatt *stbuf)
-{
-        afr_private_t *priv          = NULL;
-        afr_local_t   *impunge_local = NULL;
-        dict_t        *dict          = NULL;
-        int            ret           = 0;
-
-        priv = this->private;
-        impunge_local = impunge_frame->local;
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "creating missing file %s on %s",
-                impunge_local->loc.path,
-                priv->children[child_index]->name);
-
-        dict = dict_new ();
-        if (!dict)
-                gf_log (this->name, GF_LOG_ERROR, "Out of memory");
-
-        GF_ASSERT (!uuid_is_null (stbuf->ia_gfid));
-        ret = afr_set_dict_gfid (dict, stbuf->ia_gfid);
-        if (ret)
-                gf_log (this->name, GF_LOG_INFO, "%s: gfid set failed",
-                        impunge_local->loc.path);
-
-        STACK_WIND_COOKIE (impunge_frame, afr_sh_entry_impunge_newfile_cbk,
-                           (void *) (long) child_index,
-                           priv->children[child_index],
-                           priv->children[child_index]->fops->mknod,
-                           &impunge_local->loc,
-                           st_mode_from_ia (stbuf->ia_prot, stbuf->ia_type),
-                           makedev (ia_major (stbuf->ia_rdev),
-                                    ia_minor (stbuf->ia_rdev)), 0, dict);
-
-        if (dict)
-                dict_unref (dict);
-
-        return 0;
+	if (source < 0)
+		ret = __afr_selfheal_merge_dirent (frame, this, fd, name, inode,
+						   sources, healed_sinks,
+						   locked_on, replies);
+	else
+		ret = __afr_selfheal_heal_dirent (frame, this, fd, name, inode,
+						  source, sources, healed_sinks,
+						  locked_on, replies);
+	return ret;
 }
 
 
-
-int
-afr_sh_entry_impunge_mkdir (call_frame_t *impunge_frame, xlator_t *this,
-                            int child_index, struct iatt *stbuf)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *impunge_local = NULL;
-        dict_t          *dict = NULL;
-
-        int ret = 0;
-
-        priv = this->private;
-        impunge_local = impunge_frame->local;
-
-        dict = dict_new ();
-        if (!dict) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory");
-                return 0;
-        }
-
-        GF_ASSERT (!uuid_is_null (stbuf->ia_gfid));
-        ret = afr_set_dict_gfid (dict, stbuf->ia_gfid);
-        if (ret)
-                gf_log (this->name, GF_LOG_INFO, "%s: gfid set failed",
-                        impunge_local->loc.path);
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "creating missing directory %s on %s",
-                impunge_local->loc.path,
-                priv->children[child_index]->name);
-
-        STACK_WIND_COOKIE (impunge_frame, afr_sh_entry_impunge_newfile_cbk,
-                           (void *) (long) child_index,
-                           priv->children[child_index],
-                           priv->children[child_index]->fops->mkdir,
-                           &impunge_local->loc,
-                           st_mode_from_ia (stbuf->ia_prot, stbuf->ia_type),
-                           0, dict);
-
-        if (dict)
-                dict_unref (dict);
-
-        return 0;
+static int
+afr_selfheal_entry_dirent (call_frame_t *frame, xlator_t *this, fd_t *fd,
+			   int source, unsigned char *sources,
+			   unsigned char *healed_sinks, char *name)
+{
+	afr_private_t *priv = NULL;
+	int ret = 0;
+	unsigned char *locked_on = NULL;
+	struct afr_reply *replies = NULL;
+	inode_t *inode = NULL;
+
+	priv = this->private;
+
+	locked_on = alloca0 (priv->child_count);
+
+	replies = alloca0 (priv->child_count * sizeof(*replies));
+
+	ret = afr_selfheal_entrylk (frame, this, fd->inode, this->name,
+				    name, locked_on);
+	{
+		if (ret < 2) {
+			ret = -ENOTCONN;
+			goto unlock;
+		}
+
+		inode = afr_selfheal_unlocked_lookup_on (frame, fd->inode, name,
+							 replies, locked_on);
+		if (!inode) {
+			ret = -ENOMEM;
+			goto unlock;
+		}
+
+		ret = __afr_selfheal_entry_dirent (frame, this, fd, name, inode,
+						   source, sources, healed_sinks,
+						   locked_on, replies);
+	}
+unlock:
+	afr_selfheal_unentrylk (frame, this, fd->inode, this->name, name,
+				locked_on);
+	if (inode)
+		inode_unref (inode);
+	return ret;
 }
 
 
-int
-afr_sh_entry_impunge_symlink (call_frame_t *impunge_frame, xlator_t *this,
-                              int child_index, const char *linkname)
+static int
+afr_selfheal_entry_do_subvol (call_frame_t *frame, xlator_t *this, fd_t *fd,
+			      int child, int source, unsigned char *sources,
+			      unsigned char *healed_sinks)
 {
-        afr_private_t   *priv          = NULL;
-        afr_local_t     *impunge_local = NULL;
-        dict_t          *dict          = NULL;
-        struct iatt     *buf           = NULL;
-        int              ret           = 0;
-
-        priv = this->private;
-        impunge_local = impunge_frame->local;
-
-        buf = &impunge_local->cont.symlink.buf;
-
-        dict = dict_new ();
-        if (!dict) {
-                afr_sh_entry_call_impunge_done (impunge_frame, this,
-                                                -1, ENOMEM);
-                goto out;
-        }
-
-        GF_ASSERT (!uuid_is_null (buf->ia_gfid));
-        ret = afr_set_dict_gfid (dict, buf->ia_gfid);
-        if (ret)
-                gf_log (this->name, GF_LOG_INFO,
-                        "%s: dict set gfid failed",
-                        impunge_local->loc.path);
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "creating missing symlink %s -> %s on %s",
-                impunge_local->loc.path, linkname,
-                priv->children[child_index]->name);
-
-        STACK_WIND_COOKIE (impunge_frame, afr_sh_entry_impunge_newfile_cbk,
-                           (void *) (long) child_index,
-                           priv->children[child_index],
-                           priv->children[child_index]->fops->symlink,
-                           linkname, &impunge_local->loc, 0, dict);
-
-        if (dict)
-                dict_unref (dict);
-out:
-        return 0;
-}
+	int ret = 0;
+	gf_dirent_t entries;
+	gf_dirent_t *entry = NULL;
+	off_t offset = 0;
+	call_frame_t *iter_frame = NULL;
+	xlator_t *subvol = NULL;
+	afr_private_t *priv = NULL;
 
+	priv = this->private;
+	subvol = priv->children[child];
 
-int
-afr_sh_entry_impunge_symlink_unlink_cbk (call_frame_t *impunge_frame,
-                                         void *cookie, xlator_t *this,
-                                         int32_t op_ret, int32_t op_errno,
-                                         struct iatt *preparent,
-                                         struct iatt *postparent, dict_t *xdata)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *impunge_local = NULL;
-        afr_self_heal_t *impunge_sh = NULL;
-        int              child_index = -1;
-        int              call_count = -1;
-
-        priv          = this->private;
-        impunge_local = impunge_frame->local;
-        impunge_sh    = &impunge_local->self_heal;
-
-        child_index = (long) cookie;
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "unlink of %s on %s failed (%s)",
-                        impunge_local->loc.path,
-                        priv->children[child_index]->name,
-                        strerror (op_errno));
-                goto out;
-        }
-
-        afr_sh_entry_impunge_symlink (impunge_frame, this, child_index,
-                                      impunge_sh->linkname);
-
-        return 0;
-out:
-        LOCK (&impunge_frame->lock);
-        {
-                call_count = --impunge_local->call_count;
-        }
-        UNLOCK (&impunge_frame->lock);
+	INIT_LIST_HEAD (&entries.list);
 
-        if (call_count == 0)
-                afr_sh_entry_call_impunge_done (impunge_frame, this,
-                                                op_ret, op_errno);
+	iter_frame = afr_copy_frame (frame);
+	if (!iter_frame)
+		return -ENOMEM;
 
-        return 0;
-}
+	while ((ret = syncop_readdir (subvol, fd, 131072, offset, &entries))) {
+		if (ret > 0)
+			ret = 0;
+		list_for_each_entry (entry, &entries.list, list) {
+			offset = entry->d_off;
 
+			if (!strcmp (entry->d_name, ".") ||
+			    !strcmp (entry->d_name, ".."))
+				continue;
 
-int
-afr_sh_entry_impunge_symlink_unlink (call_frame_t *impunge_frame, xlator_t *this,
-                                     int child_index)
-{
-        afr_private_t   *priv          = NULL;
-        afr_local_t     *impunge_local = NULL;
+			if (__is_root_gfid (fd->inode->gfid) &&
+			    !strcmp (entry->d_name, GF_REPLICATE_TRASH_DIR))
+				continue;
 
-        priv          = this->private;
-        impunge_local = impunge_frame->local;
+			ret = afr_selfheal_entry_dirent (iter_frame, this, fd,
+							 source, sources,
+							 healed_sinks,
+							 entry->d_name);
+			AFR_STACK_RESET (iter_frame);
 
-        gf_log (this->name, GF_LOG_DEBUG,
-                "unlinking symlink %s with wrong target on %s",
-                impunge_local->loc.path,
-                priv->children[child_index]->name);
+			if (ret)
+				break;
+		}
 
-        STACK_WIND_COOKIE (impunge_frame, afr_sh_entry_impunge_symlink_unlink_cbk,
-                           (void *) (long) child_index,
-                           priv->children[child_index],
-                           priv->children[child_index]->fops->unlink,
-                           &impunge_local->loc, 0, NULL);
+		gf_dirent_free (&entries);
+		if (ret)
+			break;
+	}
 
-        return 0;
+	AFR_STACK_DESTROY (iter_frame);
+	return ret;
 }
 
-
-int
-afr_sh_entry_impunge_readlink_sink_cbk (call_frame_t *impunge_frame, void *cookie,
-                                        xlator_t *this,
-                                        int32_t op_ret, int32_t op_errno,
-                                        const char *linkname, struct iatt *sbuf, dict_t *xdata)
+static int
+afr_selfheal_entry_do (call_frame_t *frame, xlator_t *this, fd_t *fd,
+		       int source, unsigned char *sources,
+		       unsigned char *healed_sinks,
+		       struct afr_reply *locked_replies)
 {
-        afr_private_t   *priv = NULL;
-        afr_local_t     *impunge_local = NULL;
-        afr_self_heal_t *impunge_sh = NULL;
-        int              child_index = -1;
-        int              call_count = -1;
-        int              active_src = -1;
-
-        priv          = this->private;
-        impunge_local = impunge_frame->local;
-        impunge_sh    = &impunge_local->self_heal;
-        active_src    = impunge_sh->active_source;
-
-        child_index = (long) cookie;
-
-        if ((op_ret == -1) && (op_errno != ENOENT)) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "readlink of %s on %s failed (%s)",
-                        impunge_local->loc.path,
-                        priv->children[active_src]->name,
-                        strerror (op_errno));
-                goto out;
-        }
-
-        /* symlink doesn't exist on the sink */
-
-        if ((op_ret == -1) && (op_errno == ENOENT)) {
-                afr_sh_entry_impunge_symlink (impunge_frame, this,
-                                              child_index, impunge_sh->linkname);
-                return 0;
-        }
-
-
-        /* symlink exists on the sink, so check if targets match */
-
-        if (strcmp (linkname, impunge_sh->linkname) == 0) {
-                /* targets match, nothing to do */
-
-                goto out;
-        } else {
-                /*
-                 * Hah! Sneaky wolf in sheep's clothing!
-                 */
-                afr_sh_entry_impunge_symlink_unlink (impunge_frame, this,
-                                                     child_index);
-                return 0;
-        }
+	int i = 0;
+	afr_private_t *priv = NULL;
+	int ret = 0;
 
-out:
-        LOCK (&impunge_frame->lock);
-        {
-                call_count = --impunge_local->call_count;
-        }
-        UNLOCK (&impunge_frame->lock);
+	priv = this->private;
 
-        if (call_count == 0)
-                afr_sh_entry_call_impunge_done (impunge_frame, this,
-                                                op_ret, op_errno);
+	gf_log (this->name, GF_LOG_INFO, "performing entry selfheal on %s",
+		uuid_utoa (fd->inode->gfid));
 
-        return 0;
+	for (i = 0; i < priv->child_count; i++) {
+		if (i != source && !healed_sinks[i])
+			continue;
+		ret = afr_selfheal_entry_do_subvol (frame, this, fd, i, source,
+						    sources, healed_sinks);
+		if (ret)
+			break;
+	}
+	return ret;
 }
 
 
-int
-afr_sh_entry_impunge_readlink_sink (call_frame_t *impunge_frame, xlator_t *this,
-                                    int child_index)
+static int
+__afr_selfheal_entry_finalize_source (xlator_t *this, unsigned char *sources,
+				      unsigned char *sinks,
+				      unsigned char *locked_on,
+				      struct afr_reply *replies)
 {
-        afr_private_t   *priv = NULL;
-        afr_local_t     *impunge_local = NULL;
-
-        priv = this->private;
-        impunge_local = impunge_frame->local;
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "checking symlink target of %s on %s",
-                impunge_local->loc.path, priv->children[child_index]->name);
+	int i = 0;
+	afr_private_t *priv = NULL;
+	int source = -1;
+	int locked_count = 0;
+	int sources_count = 0;
+	int sinks_count = 0;
 
-        STACK_WIND_COOKIE (impunge_frame, afr_sh_entry_impunge_readlink_sink_cbk,
-                           (void *) (long) child_index,
-                           priv->children[child_index],
-                           priv->children[child_index]->fops->readlink,
-                           &impunge_local->loc, 4096, NULL);
+	priv = this->private;
 
-        return 0;
-}
+	locked_count = AFR_COUNT (locked_on, priv->child_count);
+	sources_count = AFR_COUNT (sources, priv->child_count);
+	sinks_count = AFR_COUNT (sinks, priv->child_count);
 
+	if (locked_count == sinks_count || !sources_count) {
+		return -1;
+	}
 
-int
-afr_sh_entry_impunge_readlink_cbk (call_frame_t *impunge_frame, void *cookie,
-                                   xlator_t *this,
-                                   int32_t op_ret, int32_t op_errno,
-                                   const char *linkname, struct iatt *sbuf, dict_t *xdata)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *impunge_local = NULL;
-        afr_self_heal_t *impunge_sh = NULL;
-        int              child_index = -1;
-        int              call_count = -1;
-        int              active_src = -1;
-
-        priv = this->private;
-        impunge_local = impunge_frame->local;
-        impunge_sh = &impunge_local->self_heal;
-        active_src = impunge_sh->active_source;
-
-        child_index = (long) cookie;
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "readlink of %s on %s failed (%s)",
-                        impunge_local->loc.path,
-                        priv->children[active_src]->name,
-                        strerror (op_errno));
-                goto out;
-        }
-
-        impunge_sh->linkname = gf_strdup (linkname);
-        afr_sh_entry_impunge_readlink_sink (impunge_frame, this, child_index);
-
-        return 0;
-
-out:
-        LOCK (&impunge_frame->lock);
-        {
-                call_count = --impunge_local->call_count;
-        }
-        UNLOCK (&impunge_frame->lock);
+	for (i = 0; i < priv->child_count; i++) {
+		if (sources[i]) {
+			source = i;
+			break;
+		}
+	}
 
-        if (call_count == 0)
-                afr_sh_entry_call_impunge_done (impunge_frame, this,
-                                                op_ret, op_errno);
-
-        return 0;
+	return source;
 }
 
 
-int
-afr_sh_entry_impunge_readlink (call_frame_t *impunge_frame, xlator_t *this,
-                               int child_index, struct iatt *stbuf)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *impunge_local = NULL;
-        afr_self_heal_t *impunge_sh = NULL;
-        int              active_src = -1;
-
-        priv = this->private;
-        impunge_local = impunge_frame->local;
-        impunge_sh = &impunge_local->self_heal;
-        active_src = impunge_sh->active_source;
-        impunge_local->cont.symlink.buf = *stbuf;
-
-        STACK_WIND_COOKIE (impunge_frame, afr_sh_entry_impunge_readlink_cbk,
-                           (void *) (long) child_index,
-                           priv->children[active_src],
-                           priv->children[active_src]->fops->readlink,
-                           &impunge_local->loc, 4096, NULL);
-
-        return 0;
-}
-
-int
-afr_sh_entry_impunge_create (call_frame_t *impunge_frame, xlator_t *this,
-                             int child_index)
+static int
+__afr_selfheal_entry_prepare (call_frame_t *frame, xlator_t *this, fd_t *fd,
+			      unsigned char *locked_on, unsigned char *sources,
+			      unsigned char *sinks, unsigned char *healed_sinks,
+			      struct afr_reply *replies, int *source_p)
 {
-        call_frame_t    *frame             = NULL;
-        afr_local_t     *impunge_local     = NULL;
-        afr_local_t     *local             = NULL;
-        afr_self_heal_t *impunge_sh        = NULL;
-        afr_self_heal_t *sh                = NULL;
-        afr_private_t   *priv = NULL;
-        ia_type_t       type = IA_INVAL;
-        int             active_src = 0;
-        struct iatt     *buf = NULL;
-
-        AFR_INIT_SH_FRAME_VALS (impunge_frame, impunge_local, impunge_sh,
-                                frame, local, sh);
-        active_src = impunge_sh->active_source;
-        afr_update_loc_gfids (&impunge_local->loc, &impunge_sh->entrybuf,
-                              &impunge_sh->parentbuf);
-
-        buf = &impunge_sh->entrybuf;
-        type = buf->ia_type;
-
-        switch (type) {
-        case IA_IFSOCK:
-        case IA_IFREG:
-        case IA_IFBLK:
-        case IA_IFCHR:
-        case IA_IFIFO:
-        case IA_IFLNK:
-                afr_sh_entry_impunge_check_hardlink (impunge_frame, this,
-                                                     child_index, buf);
-                break;
-        case IA_IFDIR:
-                afr_sh_entry_impunge_mkdir (impunge_frame, this,
-                                            child_index, buf);
-                break;
-        default:
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s has unknown file type on %s: 0%o",
-                        impunge_local->loc.path,
-                        priv->children[active_src]->name, type);
-                sh->impunge_done (frame, this, -1, EINVAL);
-                break;
-        }
-
-        return 0;
-}
+	int ret = -1;
+	int source = -1;
+	afr_private_t *priv = NULL;
+	int i = 0;
 
-int
-afr_sh_entry_impunge_create_file (call_frame_t *impunge_frame, xlator_t *this,
-                                  int child_index)
-{
-        call_frame_t    *frame             = NULL;
-        afr_local_t     *impunge_local     = NULL;
-        afr_local_t     *local             = NULL;
-        afr_self_heal_t *impunge_sh        = NULL;
-        afr_self_heal_t *sh                = NULL;
-        afr_private_t   *priv = NULL;
-        ia_type_t       type = IA_INVAL;
-        int             active_src = 0;
-        struct iatt     *buf = NULL;
-
-        AFR_INIT_SH_FRAME_VALS (impunge_frame, impunge_local, impunge_sh,
-                                frame, local, sh);
-        active_src = impunge_sh->active_source;
-        buf = &impunge_sh->entrybuf;
-        type = buf->ia_type;
-
-        switch (type) {
-        case IA_IFSOCK:
-        case IA_IFREG:
-        case IA_IFBLK:
-        case IA_IFCHR:
-        case IA_IFIFO:
-                afr_sh_entry_impunge_mknod (impunge_frame, this,
-                                            child_index, buf);
-                break;
-        case IA_IFLNK:
-                afr_sh_entry_impunge_readlink (impunge_frame, this,
-                                               child_index, buf);
-                break;
-        default:
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s has unknown file type on %s: 0%o",
-                        impunge_local->loc.path,
-                        priv->children[active_src]->name, type);
-                sh->impunge_done (frame, this, -1, EINVAL);
-                break;
-        }
-
-        return 0;
-}
+	priv = this->private;
 
-gf_boolean_t
-afr_sh_need_recreate (afr_self_heal_t *impunge_sh, unsigned int child,
-                      unsigned int child_count)
-{
-        gf_boolean_t    recreate = _gf_false;
+	ret = afr_selfheal_unlocked_discover (frame, fd->inode, fd->inode->gfid,
+					      replies);
+	if (ret)
+		return ret;
 
-        GF_ASSERT (impunge_sh->child_errno);
+	ret = afr_selfheal_find_direction (frame, this, replies,
+					   AFR_ENTRY_TRANSACTION,
+					   locked_on, sources, sinks);
+	if (ret)
+		return ret;
 
-        if (child == impunge_sh->active_source)
-                goto out;
+	source = __afr_selfheal_entry_finalize_source (this, sources, sinks,
+						       locked_on, replies);
+	if (source < 0) {
+		/* If source is < 0 (typically split-brain), we perform a
+		   conservative merge of entries rather than erroring out */
+	}
+	*source_p = source;
 
-        if (IA_IFLNK == impunge_sh->entrybuf.ia_type) {
-                recreate = _gf_true;
-                goto out;
-        }
+	for (i = 0; i < priv->child_count; i++)
+		/* Initialize the healed_sinks[] array optimistically to
+		   the intersection of to-be-healed (i.e sinks[]) and
+		   the list of servers which are up (i.e locked_on[]).
 
-        if (impunge_sh->child_errno[child] == ENOENT)
-                recreate = _gf_true;
-out:
-        return recreate;
-}
+		   As we encounter failures in the healing process, we
+		   will unmark the respective servers in the healed_sinks[]
+		   array.
+		*/
+		healed_sinks[i] = sinks[i] && locked_on[i];
 
-unsigned int
-afr_sh_recreate_count (afr_self_heal_t *impunge_sh, int *sources,
-                       unsigned int child_count)
-{
-        int             count = 0;
-        int             i = 0;
-
-        for (i = 0; i < child_count; i++) {
-                if (afr_sh_need_recreate (impunge_sh, i, child_count))
-                        count++;
-        }
-
-        return count;
+	return ret;
 }
 
-int
-afr_sh_entry_call_impunge_recreate (call_frame_t *impunge_frame,
-                                    xlator_t *this)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *impunge_local = NULL;
-        afr_self_heal_t *impunge_sh = NULL;
-        call_frame_t    *frame = NULL;
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        unsigned int     recreate_count = 0;
-        int              i = 0;
-        int              active_src = 0;
-
-        priv          = this->private;
-        AFR_INIT_SH_FRAME_VALS (impunge_frame, impunge_local, impunge_sh,
-                                frame, local, sh);
-        active_src = impunge_sh->active_source;
-        impunge_sh->entrybuf = impunge_sh->buf[active_src];
-        impunge_sh->parentbuf = impunge_sh->parentbufs[active_src];
-        recreate_count = afr_sh_recreate_count (impunge_sh, sh->sources,
-                                                priv->child_count);
-        if (!recreate_count) {
-                afr_sh_entry_call_impunge_done (impunge_frame, this, 0, 0);
-                goto out;
-        }
-        impunge_local->call_count = recreate_count;
-        for (i = 0; i < priv->child_count; i++) {
-                if (!impunge_local->child_up[i]) {
-                        impunge_sh->child_errno[i] = ENOTCONN;
-                        continue;
-                }
-                if (!afr_sh_need_recreate (impunge_sh, i, priv->child_count)) {
-                        impunge_sh->child_errno[i] = EEXIST;
-                        continue;
-                }
-        }
-        for (i = 0; i < priv->child_count; i++) {
-                if (!afr_sh_need_recreate (impunge_sh, i, priv->child_count))
-                        continue;
-                (void)afr_sh_entry_impunge_create (impunge_frame, this, i);
-                recreate_count--;
-        }
-        GF_ASSERT (!recreate_count);
-out:
-        return 0;
-}
 
-void
-afr_sh_entry_common_lookup_done (call_frame_t *impunge_frame, xlator_t *this,
-                                 int32_t op_ret, int32_t op_errno)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *impunge_local = NULL;
-        afr_self_heal_t *impunge_sh = NULL;
-        call_frame_t    *frame = NULL;
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        unsigned int     gfid_miss_count = 0;
-        unsigned int     children_up_count = 0;
-        uuid_t           gfid = {0};
-        int              active_src = 0;
-
-        priv          = this->private;
-        AFR_INIT_SH_FRAME_VALS (impunge_frame, impunge_local, impunge_sh,
-                                frame, local, sh);
-        active_src    = impunge_sh->active_source;
-
-        if (op_ret < 0)
-                goto done;
-        if (impunge_sh->child_errno[active_src]) {
-                op_ret = -1;
-                op_errno = impunge_sh->child_errno[active_src];
-                goto done;
-        }
-
-        gfid_miss_count = afr_gfid_missing_count (this->name,
-                                                  impunge_sh->success_children,
-                                                  impunge_sh->buf, priv->child_count,
-                                                  impunge_local->loc.path);
-        children_up_count = afr_up_children_count (impunge_local->child_up,
-                                                   priv->child_count);
-        if ((gfid_miss_count == children_up_count) &&
-            (children_up_count < priv->child_count)) {
-                op_ret = -1;
-                op_errno = ENODATA;
-                gf_log (this->name, GF_LOG_ERROR, "Not all children are up, "
-                        "gfid should not be assigned in this state for %s",
-                        impunge_local->loc.path);
-                goto done;
-        }
-
-        if (gfid_miss_count) {
-                afr_update_gfid_from_iatts (gfid, impunge_sh->buf,
-                                            impunge_sh->success_children,
-                                            priv->child_count);
-                if (uuid_is_null (gfid)) {
-                        sh->entries_skipped = _gf_true;
-                        gf_log (this->name, GF_LOG_INFO, "%s: Skipping entry "
-                                "self-heal because of gfid absence",
-                                impunge_local->loc.path);
-                        goto done;
-                }
-                afr_sh_common_lookup (impunge_frame, this, &impunge_local->loc,
-                                      afr_sh_entry_common_lookup_done, gfid,
-                                      AFR_LOOKUP_FAIL_CONFLICTS |
-                                      AFR_LOOKUP_FAIL_MISSING_GFIDS,
-                                      NULL);
-        } else {
-                afr_sh_entry_call_impunge_recreate (impunge_frame, this);
-        }
-        return;
-done:
-        afr_sh_entry_call_impunge_done (impunge_frame, this,
-                                        op_ret, op_errno);
-        return;
-}
-
-int
-afr_sh_entry_impunge_entry (call_frame_t *frame, xlator_t *this,
-                            gf_dirent_t *entry)
-{
-        afr_local_t     *local  = NULL;
-        afr_self_heal_t *sh  = NULL;
-        afr_self_heal_t *impunge_sh  = NULL;
-        int              ret = -1;
-        call_frame_t    *impunge_frame = NULL;
-        afr_local_t     *impunge_local = NULL;
-        int              active_src = 0;
-        int              op_errno = 0;
-        int              op_ret = -1;
-
-        local = frame->local;
-        sh = &local->self_heal;
-
-        active_src = sh->active_source;
-        sh->impunge_done = afr_sh_entry_impunge_entry_done;
-
-        if ((strcmp (entry->d_name, ".") == 0)
-            || (strcmp (entry->d_name, "..") == 0)
-            || ((strcmp (local->loc.path, "/") == 0)
-                && (strcmp (entry->d_name, GF_REPLICATE_TRASH_DIR) == 0))) {
-
-                gf_log (this->name, GF_LOG_TRACE,
-                        "skipping inspection of %s under %s",
-                        entry->d_name, local->loc.path);
-                op_ret = 0;
-                goto out;
-        }
-
-        gf_log (this->name, GF_LOG_TRACE,
-                "inspecting existence of %s under %s",
-                entry->d_name, local->loc.path);
-
-        ret = afr_impunge_frame_create (frame, this, active_src,
-                                        &impunge_frame);
-        if (ret) {
-                op_errno = -ret;
-                goto out;
-        }
-
-        impunge_local = impunge_frame->local;
-        impunge_sh = &impunge_local->self_heal;
-        ret = afr_build_child_loc (this, &impunge_local->loc, &local->loc,
-                                   entry->d_name);
-        loc_copy (&impunge_sh->parent_loc, &local->loc);
-        if (ret != 0) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        afr_sh_common_lookup (impunge_frame, this, &impunge_local->loc,
-                              afr_sh_entry_common_lookup_done, NULL,
-                              AFR_LOOKUP_FAIL_CONFLICTS, NULL);
-
-        op_ret = 0;
+static int
+__afr_selfheal_entry (call_frame_t *frame, xlator_t *this, fd_t *fd,
+		      unsigned char *locked_on)
+{
+	afr_private_t *priv = NULL;
+	int ret = -1;
+	unsigned char *sources = NULL;
+	unsigned char *sinks = NULL;
+	unsigned char *data_lock = NULL;
+	unsigned char *healed_sinks = NULL;
+	struct afr_reply *locked_replies = NULL;
+	int source = -1;
+
+	priv = this->private;
+
+	sources = alloca0 (priv->child_count);
+	sinks = alloca0 (priv->child_count);
+	healed_sinks = alloca0 (priv->child_count);
+	data_lock = alloca0 (priv->child_count);
+
+	locked_replies = alloca0 (sizeof (*locked_replies) * priv->child_count);
+
+	ret = afr_selfheal_entrylk (frame, this, fd->inode, this->name, NULL,
+				    data_lock);
+	{
+		if (ret < 2) {
+			ret = -ENOTCONN;
+			goto unlock;
+		}
+
+		ret = __afr_selfheal_entry_prepare (frame, this, fd, data_lock,
+						    sources, sinks, healed_sinks,
+						    locked_replies, &source);
+	}
+unlock:
+	afr_selfheal_unentrylk (frame, this, fd->inode, this->name, NULL,
+				data_lock);
+	if (ret < 0)
+		goto out;
+
+	ret = afr_selfheal_entry_do (frame, this, fd, source, sources,
+				     healed_sinks, locked_replies);
+	if (ret)
+		goto out;
+
+	ret = afr_selfheal_undo_pending (frame, this, fd->inode, sources, sinks,
+					 healed_sinks, AFR_ENTRY_TRANSACTION,
+					 locked_replies, data_lock);
 out:
-        if (ret) {
-                if (impunge_frame)
-                        AFR_STACK_DESTROY (impunge_frame);
-                sh->impunge_done (frame, this, op_ret, op_errno);
-        }
-
-        return 0;
-}
-
-
-int
-afr_sh_entry_impunge_readdir_cbk (call_frame_t *frame, void *cookie,
-                                  xlator_t *this,
-                                  int32_t op_ret, int32_t op_errno,
-                                  gf_dirent_t *entries, dict_t *xdata)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *local  = NULL;
-        afr_self_heal_t *sh  = NULL;
-        gf_dirent_t     *entry = NULL;
-        off_t            last_offset = 0;
-        int              active_src = 0;
-        int              entry_count = 0;
-
-        priv = this->private;
-        local = frame->local;
-        sh = &local->self_heal;
-
-        active_src = sh->active_source;
-
-        if (op_ret <= 0) {
-                if (op_ret < 0) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "readdir of %s on subvolume %s failed (%s)",
-                                local->loc.path,
-                                priv->children[active_src]->name,
-                                strerror (op_errno));
-                        sh->op_failed = 1;
-                } else {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "readdir of %s on subvolume %s complete",
-                                local->loc.path,
-                                priv->children[active_src]->name);
-                }
-
-                afr_sh_entry_impunge_all (frame, this);
-                return 0;
-        }
-
-        list_for_each_entry (entry, &entries->list, list) {
-                last_offset = entry->d_off;
-                entry_count++;
-        }
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "readdir'ed %d entries from %s",
-                entry_count, priv->children[active_src]->name);
-
-        sh->offset = last_offset;
-        local->call_count = entry_count;
-
-        list_for_each_entry (entry, &entries->list, list) {
-                afr_sh_entry_impunge_entry (frame, this, entry);
-        }
-
-        return 0;
+	return ret;
 }
 
 
-int
-afr_sh_entry_impunge_subvol (call_frame_t *frame, xlator_t *this)
+static fd_t *
+afr_selfheal_data_opendir (xlator_t *this, inode_t *inode)
 {
-        afr_private_t   *priv = NULL;
-        afr_local_t     *local  = NULL;
-        afr_self_heal_t *sh  = NULL;
-        int32_t         active_src = 0;
-
-        priv = this->private;
-        local = frame->local;
-        sh = &local->self_heal;
-        active_src = sh->active_source;
-        gf_log (this->name, GF_LOG_DEBUG, "%s: readdir from offset %zd",
-                local->loc.path, sh->offset);
-
-        STACK_WIND (frame, afr_sh_entry_impunge_readdir_cbk,
-                    priv->children[active_src],
-                    priv->children[active_src]->fops->readdirp,
-                    sh->healing_fd, sh->block_size, sh->offset, NULL);
-
-        return 0;
-}
-
-
-int
-afr_sh_entry_impunge_all (call_frame_t *frame, xlator_t *this)
-{
-        afr_private_t   *priv = NULL;
-        afr_local_t     *local  = NULL;
-        afr_self_heal_t *sh  = NULL;
-        int              active_src = -1;
-
-        priv = this->private;
-        local = frame->local;
-        sh = &local->self_heal;
-
-        sh->offset = 0;
+	loc_t loc = {0,};
+	int ret = 0;
+	fd_t *fd = NULL;
 
-        active_src = next_active_source (frame, this, sh->active_source);
-        sh->active_source = active_src;
+	fd = fd_create (inode, 0);
+	if (!fd)
+		return NULL;
 
-        if (sh->op_failed) {
-                afr_sh_entry_finish (frame, this);
-                return 0;
-        }
+	loc.inode = inode_ref (inode);
+	uuid_copy (loc.gfid, inode->gfid);
 
-        if (active_src == -1) {
-                /* completed creating missing files on all subvolumes */
-                afr_sh_entry_erase_pending (frame, this);
-                return 0;
-        }
+	ret = syncop_opendir (this, &loc, fd);
+	if (ret) {
+		fd_unref (fd);
+		fd = NULL;
+	} else {
+		fd_bind (fd);
+	}
 
-        gf_log (this->name, GF_LOG_TRACE,
-                "impunging entries of %s on %s to other sinks",
-                local->loc.path, priv->children[active_src]->name);
+	loc_wipe (&loc);
 
-        afr_sh_entry_impunge_subvol (frame, this);
-
-        return 0;
+	return fd;
 }
 
 
-int
-afr_sh_entry_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
-{
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        int              call_count = 0;
-        int              child_index = 0;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        child_index = (long) cookie;
-
-        /* TODO: some of the open's might fail.
-           In that case, modify cleanup fn to send flush on those
-           fd's which are already open */
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "opendir of %s failed on child %s (%s)",
-                                local->loc.path,
-                                priv->children[child_index]->name,
-                                strerror (op_errno));
-                        sh->op_failed = 1;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                if (sh->op_failed) {
-                        afr_sh_entry_finish (frame, this);
-                        return 0;
-                }
-                gf_log (this->name, GF_LOG_TRACE,
-                        "fd for %s opened, commencing sync",
-                        local->loc.path);
-
-                sh->active_source = -1;
-                afr_sh_entry_expunge_all (frame, this);
-        }
-
-        return 0;
-}
-
-
-int
-afr_sh_entry_open (call_frame_t *frame, xlator_t *this)
-{
-        int i = 0;
-        int call_count = 0;
-
-        int source = -1;
-        int *sources = NULL;
-
-        fd_t *fd = NULL;
-
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
-        afr_self_heal_t *sh = NULL;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        source  = local->self_heal.source;
-        sources = local->self_heal.sources;
-
-        sh->block_size = 65536; //131072
-        sh->offset = 0;
-
-        call_count = sh->active_sinks;
-        if (source != -1)
-                call_count++;
-
-        local->call_count = call_count;
-
-        fd = fd_create (local->loc.inode, frame->root->pid);
-        sh->healing_fd = fd;
-
-        if (source != -1) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "opening directory %s on subvolume %s (source)",
-                        local->loc.path, priv->children[source]->name);
-
-                /* open source */
-                STACK_WIND_COOKIE (frame, afr_sh_entry_opendir_cbk,
-                                   (void *) (long) source,
-                                   priv->children[source],
-                                   priv->children[source]->fops->opendir,
-                                   &local->loc, fd, NULL);
-                call_count--;
-        }
-
-        /* open sinks */
-        for (i = 0; i < priv->child_count; i++) {
-                if (sources[i] || !local->child_up[i])
-                        continue;
-
-                gf_log (this->name, GF_LOG_TRACE,
-                        "opening directory %s on subvolume %s (sink)",
-                        local->loc.path, priv->children[i]->name);
-
-                STACK_WIND_COOKIE (frame, afr_sh_entry_opendir_cbk,
-                                   (void *) (long) i,
-                                   priv->children[i],
-                                   priv->children[i]->fops->opendir,
-                                   &local->loc, fd, NULL);
-
-                if (!--call_count)
-                        break;
-        }
-
-        return 0;
-}
-
 
 int
-afr_sh_entry_sync_prepare (call_frame_t *frame, xlator_t *this)
+afr_selfheal_entry (call_frame_t *frame, xlator_t *this, inode_t *inode)
 {
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        int              source = 0;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        source = sh->source;
-
-        afr_sh_mark_source_sinks (frame, this);
-        if (source != -1)
-                sh->success[source] = 1;
-
-        if (sh->active_sinks == 0) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "no active sinks for self-heal on dir %s",
-                        local->loc.path);
-                afr_sh_entry_finish (frame, this);
-                return 0;
-        }
-        if (source == -1 && sh->active_sinks < 2) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "cannot sync with 0 sources and 1 sink on dir %s",
-                        local->loc.path);
-                afr_sh_entry_finish (frame, this);
-                return 0;
-        }
-
-        if (source != -1)
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "self-healing directory %s from subvolume %s to "
-                        "%d other",
-                        local->loc.path, priv->children[source]->name,
-                        sh->active_sinks);
-        else
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "no active sources for %s found. "
-                        "merging all entries as a conservative decision",
-                        local->loc.path);
-
-        afr_sh_entry_open (frame, this);
-
-        return 0;
-}
+	afr_private_t *priv = NULL;
+	unsigned char *locked_on = NULL;
+	fd_t *fd = NULL;
+	int ret = 0;
 
+	priv = this->private;
 
-void
-afr_sh_entry_fix (call_frame_t *frame, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno)
-{
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        int              source = 0;
-        int              nsources = 0;
-        int32_t          subvol_status = 0;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        if (op_ret < 0) {
-                sh->op_failed = 1;
-                afr_sh_set_error (sh, op_errno);
-                afr_sh_entry_finish (frame, this);
-                goto out;
-        }
-
-        if (sh->forced_merge) {
-                sh->source = -1;
-                goto heal;
-        }
-
-        nsources = afr_build_sources (this, sh->xattr, sh->buf,
-                                      sh->pending_matrix, sh->sources,
-                                      sh->success_children,
-                                      AFR_ENTRY_TRANSACTION, &subvol_status,
-                                      _gf_true);
-        if ((subvol_status & ALL_FOOLS) ||
-            (subvol_status & SPLIT_BRAIN)) {
-                gf_log (this->name, GF_LOG_INFO, "%s: Performing conservative "
-                        "merge", local->loc.path);
-                source = -1;
-                memset (sh->sources, 0,
-                        sizeof (*sh->sources) * priv->child_count);
-        } else if (nsources == 0) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "No self-heal needed for %s",
-                        local->loc.path);
-
-                afr_sh_entry_finish (frame, this);
-                return;
-        } else {
-                source = afr_sh_select_source (sh->sources, priv->child_count);
-        }
-
-        sh->source = source;
-
-        afr_reset_children (sh->fresh_children, priv->child_count);
-        afr_get_fresh_children (sh->success_children, sh->sources,
-                                sh->fresh_children, priv->child_count);
-        if (sh->source >= 0)
-                afr_inode_set_read_ctx (this, sh->inode, sh->source,
-                                        sh->fresh_children);
-
-heal:
-        afr_sh_entry_sync_prepare (frame, this);
-out:
-        return;
-}
+	fd = afr_selfheal_data_opendir (this, inode);
+	if (!fd)
+		return -EIO;
 
-int
-afr_sh_post_nonblocking_entry_cbk (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_self_heal_t     *sh       = NULL;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        sh       = &local->self_heal;
-
-        if (int_lock->lock_op_ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Non Blocking entrylks "
-                        "failed for %s.", local->loc.path);
-                sh->op_failed = 1;
-                afr_sh_entry_done (frame, this);
-        } else {
-
-                gf_log (this->name, GF_LOG_DEBUG, "Non Blocking entrylks done "
-                        "for %s. Proceeding to FOP", local->loc.path);
-                afr_sh_common_lookup (frame, this, &local->loc,
-                                      afr_sh_entry_fix, NULL,
-                                      AFR_LOOKUP_FAIL_CONFLICTS |
-                                      AFR_LOOKUP_FAIL_MISSING_GFIDS,
-                                      NULL);
-        }
-
-        return 0;
-}
-
-int
-afr_self_heal_entry (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t   *local = NULL;
-        afr_private_t   *priv = NULL;
+	locked_on = alloca0 (priv->child_count);
 
+	ret = afr_selfheal_tryentrylk (frame, this, inode, priv->sh_domain, NULL,
+				       locked_on);
+	{
+		if (ret < 2) {
+			/* Either less than two subvols available, or another
+			   selfheal (from another server) is in progress. Skip
+			   for now in any case there isn't anything to do.
+			*/
+			ret = -ENOTCONN;
+			goto unlock;
+		}
 
-        priv = this->private;
-        local = frame->local;
+		ret = __afr_selfheal_entry (frame, this, fd, locked_on);
+	}
+unlock:
+	afr_selfheal_unentrylk (frame, this, inode, priv->sh_domain, NULL, locked_on);
 
-        if (local->self_heal.do_entry_self_heal && priv->entry_self_heal) {
-                afr_sh_entrylk (frame, this, &local->loc, NULL,
-                                afr_sh_post_nonblocking_entry_cbk);
-        } else {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "proceeding to completion on %s",
-                        local->loc.path);
-                afr_sh_entry_done (frame, this);
-        }
+	if (fd)
+		fd_unref (fd);
 
-        return 0;
+	return ret;
 }
diff --git a/xlators/cluster/afr/src/afr-self-heal-metadata.c b/xlators/cluster/afr/src/afr-self-heal-metadata.c
index 7611088db..b31a33237 100644
--- a/xlators/cluster/afr/src/afr-self-heal-metadata.c
+++ b/xlators/cluster/afr/src/afr-self-heal-metadata.c
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -8,601 +8,274 @@
   cases as published by the Free Software Foundation.
 */
 
-#include <libgen.h>
-#include <unistd.h>
-#include <fnmatch.h>
-#include <sys/time.h>
-#include <stdlib.h>
-#include <signal.h>
 
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
 #endif
 
-#include "glusterfs.h"
 #include "afr.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "stack.h"
-#include "list.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include "compat-errno.h"
-#include "compat.h"
-#include "byte-order.h"
-
-#include "afr-transaction.h"
 #include "afr-self-heal.h"
-#include "afr-self-heal-common.h"
-
-
-int
-afr_sh_metadata_done (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-
-        local = frame->local;
-        sh = &local->self_heal;
-
-        afr_sh_reset (frame, this);
-        if (sh->mdata_spb) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "split-brain detected, aborting selfheal of %s",
-                        local->loc.path);
-                sh->op_failed = 1;
-                sh->completion_cbk (frame, this);
-        } else {
-                if (IA_ISDIR (sh->type)) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "proceeding to entry check on %s",
-                                local->loc.path);
-                        afr_self_heal_entry (frame, this);
-                        return 0;
-                }
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "proceeding to data check on %s",
-                        local->loc.path);
-                afr_self_heal_data (frame, this);
-        }
-
-        return 0;
-}
-
-int
-afr_sh_inode_unlock (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        int_lock->lock_cbk = afr_sh_metadata_done;
-        afr_unlock (frame, this);
-
-        return 0;
-}
-
-int
-afr_sh_metadata_finish (call_frame_t *frame, xlator_t *this)
-{
-        afr_sh_inode_unlock (frame, this);
-
-        return 0;
-}
-
-
-int
-afr_sh_metadata_erase_pending_cbk (call_frame_t *frame, void *cookie,
-                                   xlator_t *this, int32_t op_ret,
-                                   int32_t op_errno, dict_t *xattr, dict_t *xdata)
-{
-        afr_local_t     *local     = NULL;
-        int             call_count = 0;
-        long            i          = 0;
-        afr_self_heal_t *sh        = NULL;
-        afr_private_t   *priv      = NULL;
-
-        local = frame->local;
-        priv  = this->private;
-        sh = &local->self_heal;
-        i = (long)cookie;
-
-        if ((!IA_ISREG (sh->buf[sh->source].ia_type)) &&
-            (!IA_ISDIR (sh->buf[sh->source].ia_type))) {
-                afr_children_add_child (sh->fresh_children, i,
-                                        priv->child_count);
-        }
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                if ((!IA_ISREG (sh->buf[sh->source].ia_type)) &&
-                    (!IA_ISDIR (sh->buf[sh->source].ia_type))) {
-                        afr_inode_set_read_ctx (this, sh->inode, sh->source,
-                                                sh->fresh_children);
-                }
-                afr_sh_metadata_finish (frame, this);
-        }
-
-        return 0;
-}
-
-
-int
-afr_sh_metadata_erase_pending (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        int              call_count = 0;
-        int              i = 0;
-        dict_t          **erase_xattr = NULL;
-
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        afr_sh_pending_to_delta (priv, sh->xattr, sh->delta_matrix,
-                                 sh->success, priv->child_count,
-                                 AFR_METADATA_TRANSACTION);
-
-        erase_xattr = GF_CALLOC (sizeof (*erase_xattr), priv->child_count,
-                                 gf_afr_mt_dict_t);
-        if (!erase_xattr)
-                return -ENOMEM;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (sh->xattr[i]) {
-                        call_count++;
-
-                        erase_xattr[i] = get_new_dict();
-                        dict_ref (erase_xattr[i]);
-                }
-        }
-
-        afr_sh_delta_to_xattr (priv, sh->delta_matrix, erase_xattr,
-                               priv->child_count, AFR_METADATA_TRANSACTION);
-
-        local->call_count = call_count;
-
-        if (call_count == 0) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "metadata of %s not healed on any subvolume",
-                        local->loc.path);
-
-                afr_sh_metadata_finish (frame, this);
-        }
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (!erase_xattr[i])
-                        continue;
-
-                gf_log (this->name, GF_LOG_TRACE,
-                        "erasing pending flags from %s on %s",
-                        local->loc.path, priv->children[i]->name);
-
-                STACK_WIND_COOKIE (frame, afr_sh_metadata_erase_pending_cbk,
-                                   (void *) (long) i,
-                                   priv->children[i],
-                                   priv->children[i]->fops->xattrop,
-                                   &local->loc,
-                                   GF_XATTROP_ADD_ARRAY, erase_xattr[i],
-                                   NULL);
-                if (!--call_count)
-                        break;
-        }
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (erase_xattr[i]) {
-                        dict_unref (erase_xattr[i]);
-                }
-        }
-        GF_FREE (erase_xattr);
-
-        return 0;
-}
-
-
-int
-afr_sh_metadata_sync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        int              call_count = 0;
-        int              child_index = 0;
-
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        child_index = (long) cookie;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "setting attributes failed for %s on %s (%s)",
-                                local->loc.path,
-                                priv->children[child_index]->name,
-                                strerror (op_errno));
-
-                        sh->success[child_index] = 0;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0)
-                afr_sh_metadata_erase_pending (frame, this);
+#include "byte-order.h"
 
-        return 0;
-}
 
+#define AFR_HEAL_ATTR (GF_SET_ATTR_UID|GF_SET_ATTR_GID|GF_SET_ATTR_MODE)
 
 int
-afr_sh_metadata_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                             int32_t op_ret, int32_t op_errno,
-                             struct iatt *preop, struct iatt *postop, dict_t *xdata)
+afr_selfheal_metadata_do (call_frame_t *frame, xlator_t *this, inode_t *inode,
+			  int source, unsigned char *healed_sinks,
+			  struct afr_reply *locked_replies)
 {
-        afr_sh_metadata_sync_cbk (frame, cookie, this, op_ret, op_errno, xdata);
-
-        return 0;
+	int ret = -1;
+	loc_t loc = {0,};
+	dict_t *xattr = NULL;
+	dict_t *old_xattr = NULL;
+	afr_private_t *priv = NULL;
+	int i = 0;
+
+	priv = this->private;
+
+	loc.inode = inode_ref (inode);
+	uuid_copy (loc.gfid, inode->gfid);
+
+	gf_log (this->name, GF_LOG_INFO, "performing metadata selfheal on %s",
+		uuid_utoa (inode->gfid));
+
+	ret = syncop_getxattr (priv->children[source], &loc, &xattr, NULL);
+	if (ret < 0) {
+		loc_wipe (&loc);
+		return -EIO;
+	}
+
+	afr_filter_xattrs (xattr);
+	dict_del (xattr, GF_SELINUX_XATTR_KEY);
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!healed_sinks[i])
+			continue;
+
+		ret = syncop_setattr (priv->children[i], &loc,
+				      &locked_replies[source].poststat,
+				      AFR_HEAL_ATTR, NULL, NULL);
+		if (ret)
+			healed_sinks[i] = 0;
+
+		old_xattr = NULL;
+		ret = syncop_getxattr (priv->children[i], &loc, &old_xattr, 0);
+		if (old_xattr) {
+			dict_del (old_xattr, GF_SELINUX_XATTR_KEY);
+			afr_filter_xattrs (old_xattr);
+			ret = syncop_removexattr (priv->children[i], &loc, "",
+						  old_xattr);
+		}
+
+		ret = syncop_setxattr (priv->children[i], &loc, xattr, 0);
+		if (ret)
+			healed_sinks[i] = 0;
+	}
+
+	loc_wipe (&loc);
+	if (xattr)
+		dict_unref (xattr);
+
+	return 0;
 }
 
 
-int
-afr_sh_metadata_xattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno, dict_t *xdata)
+/*
+ * Look for mismatching uid/gid or mode even if xattrs don't say so, and
+ * pick one arbitrarily as winner.
+ */
+
+static int
+__afr_selfheal_metadata_finalize_source (xlator_t *this, unsigned char *sources,
+					 unsigned char *sinks,
+					 unsigned char *locked_on,
+					 struct afr_reply *replies)
 {
-        afr_sh_metadata_sync_cbk (frame, cookie, this, op_ret, op_errno, xdata);
-
-        return 0;
+	int i = 0;
+	afr_private_t *priv = NULL;
+	struct iatt first = {0, };
+	int source = -1;
+	int locked_count = 0;
+	int sources_count = 0;
+	int sinks_count = 0;
+
+	priv = this->private;
+
+	locked_count = AFR_COUNT (locked_on, priv->child_count);
+	sources_count = AFR_COUNT (sources, priv->child_count);
+	sinks_count = AFR_COUNT (sinks, priv->child_count);
+
+	if (locked_count == sinks_count || !sources_count) {
+		if (!priv->metadata_splitbrain_forced_heal) {
+			return -EIO;
+		}
+		/* Metadata split brain, select one subvol
+		   arbitrarily */
+		for (i = 0; i < priv->child_count; i++) {
+			if (locked_on[i] && sinks[i]) {
+				sources[i] = 1;
+				sinks[i] = 0;
+				break;
+			}
+		}
+	}
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!sources[i])
+			continue;
+		if (source == -1) {
+			source = i;
+			first = replies[i].poststat;
+		}
+	}
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!sources[i])
+			continue;
+		if (!IA_EQUAL (first, replies[i].poststat, type) ||
+		    !IA_EQUAL (first, replies[i].poststat, uid) ||
+		    !IA_EQUAL (first, replies[i].poststat, gid) ||
+		    !IA_EQUAL (first, replies[i].poststat, prot)) {
+			sources[i] = 0;
+			sinks[i] = 1;
+		}
+	}
+
+	return source;
 }
 
 
-int
-afr_sh_metadata_sync (call_frame_t *frame, xlator_t *this, dict_t *xattr)
+static int
+__afr_selfheal_metadata_prepare (call_frame_t *frame, xlator_t *this, inode_t *inode,
+				 unsigned char *locked_on, unsigned char *sources,
+				 unsigned char *sinks, unsigned char *healed_sinks,
+				 struct afr_reply *replies)
 {
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        int              source = 0;
-        int              active_sinks = 0;
-        int              call_count = 0;
-        int              i = 0;
-
-        struct iatt      stbuf = {0,};
-        int32_t          valid = 0;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        source = sh->source;
-        active_sinks = sh->active_sinks;
-
-        /*
-         * 2 calls per sink - setattr, setxattr
-         */
-        if (xattr)
-                call_count = active_sinks * 2;
-        else
-                call_count = active_sinks;
-
-        local->call_count = call_count;
-
-        stbuf.ia_atime = sh->buf[source].ia_atime;
-        stbuf.ia_atime_nsec = sh->buf[source].ia_atime_nsec;
-        stbuf.ia_mtime = sh->buf[source].ia_mtime;
-        stbuf.ia_mtime_nsec = sh->buf[source].ia_mtime_nsec;
-
-        stbuf.ia_uid = sh->buf[source].ia_uid;
-        stbuf.ia_gid = sh->buf[source].ia_gid;
-
-        stbuf.ia_type = sh->buf[source].ia_type;
-        stbuf.ia_prot = sh->buf[source].ia_prot;
-
-        valid = GF_SET_ATTR_MODE  |
-                GF_SET_ATTR_UID   | GF_SET_ATTR_GID |
-                GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (call_count == 0) {
-                        break;
-                }
-                if (sh->sources[i] || !local->child_up[i])
-                        continue;
-
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "self-healing metadata of %s from %s to %s",
-                        local->loc.path, priv->children[source]->name,
-                        priv->children[i]->name);
-
-                STACK_WIND_COOKIE (frame, afr_sh_metadata_setattr_cbk,
-                                   (void *) (long) i,
-                                   priv->children[i],
-                                   priv->children[i]->fops->setattr,
-                                   &local->loc, &stbuf, valid, NULL);
-
-                call_count--;
-
-                if (!xattr)
-                        continue;
-
-                STACK_WIND_COOKIE (frame, afr_sh_metadata_xattr_cbk,
-                                   (void *) (long) i,
-                                   priv->children[i],
-                                   priv->children[i]->fops->setxattr,
-                                   &local->loc, xattr, 0, NULL);
-                call_count--;
-        }
-
-        return 0;
+	int ret = -1;
+	int source = -1;
+	afr_private_t *priv = NULL;
+	int i = 0;
+
+	priv = this->private;
+
+	ret = afr_selfheal_unlocked_discover (frame, inode, inode->gfid,
+					      replies);
+	if (ret)
+		return ret;
+
+	ret = afr_selfheal_find_direction (frame, this, replies,
+					   AFR_METADATA_TRANSACTION,
+					   locked_on, sources, sinks);
+	if (ret)
+		return ret;
+
+	source = __afr_selfheal_metadata_finalize_source (this, sources, sinks,
+							  locked_on, replies);
+	if (source < 0)
+		return -EIO;
+
+	for (i = 0; i < priv->child_count; i++)
+		/* Initialize the healed_sinks[] array optimistically to
+		   the intersection of to-be-healed (i.e sinks[]) and
+		   the list of servers which are up (i.e locked_on[]).
+
+		   As we encounter failures in the healing process, we
+		   will unmark the respective servers in the healed_sinks[]
+		   array.
+		*/
+		healed_sinks[i] = sinks[i] && locked_on[i];
+
+	return source;
 }
 
 
-int
-afr_sh_metadata_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                              int32_t op_ret, int32_t op_errno, dict_t *xattr,
-                              dict_t *xdata)
+static int
+__afr_selfheal_metadata (call_frame_t *frame, xlator_t *this, inode_t *inode,
+			 unsigned char *locked_on)
 {
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        int              source = 0;
-
-        int i;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        source = sh->source;
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "getxattr of %s failed on subvolume %s (%s). proceeding without xattr",
-                        local->loc.path, priv->children[source]->name,
-                        strerror (op_errno));
-
-                afr_sh_metadata_sync (frame, this, NULL);
-        } else {
-                for (i = 0; i < priv->child_count; i++) {
-                        dict_del (xattr, priv->pending_key[i]);
-                }
-
-                afr_sh_metadata_sync (frame, this, xattr);
-        }
-
-        return 0;
-}
-
-
-int
-afr_sh_metadata_sync_prepare (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        int              source = 0;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        source = sh->source;
-
-        afr_sh_mark_source_sinks (frame, this);
-        if (sh->active_sinks == 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "no active sinks for performing self-heal on file %s",
-                        local->loc.path);
-                afr_sh_metadata_finish (frame, this);
-                return 0;
-        }
-
-        gf_log (this->name, GF_LOG_TRACE,
-                "syncing metadata of %s from subvolume %s to %d active sinks",
-                local->loc.path, priv->children[source]->name,
-                sh->active_sinks);
-
-        STACK_WIND (frame, afr_sh_metadata_getxattr_cbk,
-                    priv->children[source],
-                    priv->children[source]->fops->getxattr,
-                    &local->loc, NULL, NULL);
-
-        return 0;
-}
-
-
-void
-afr_sh_metadata_fix (call_frame_t *frame, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno)
-{
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        int              nsources = 0;
-        int              source = 0;
-        int              i = 0;
-
-        local = frame->local;
-        sh = &local->self_heal;
-        priv = this->private;
-
-        if (op_ret < 0) {
-                sh->op_failed = 1;
-                afr_sh_set_error (sh, op_errno);
-                afr_sh_metadata_finish (frame, this);
-                goto out;
-        }
-        nsources = afr_build_sources (this, sh->xattr, sh->buf,
-                                      sh->pending_matrix, sh->sources,
-                                      sh->success_children,
-                                      AFR_METADATA_TRANSACTION, NULL, _gf_false);
-        if ((nsources == -1)
-            && (priv->favorite_child != -1)
-            && (sh->child_errno[priv->favorite_child] == 0)) {
-
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Picking favorite child %s as authentic source to resolve conflicting metadata of %s",
-                        priv->children[priv->favorite_child]->name,
-                        local->loc.path);
-
-                sh->sources[priv->favorite_child] = 1;
-
-                nsources = afr_sh_source_count (sh->sources,
-                                                priv->child_count);
-        }
-
-        if (nsources == -1) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to self-heal permissions/ownership of '%s' "
-                        "(possible split-brain). Please fix the file on "
-                        "all backend volumes", local->loc.path);
-
-                sh->mdata_spb = _gf_true;
-
-                afr_sh_metadata_finish (frame, this);
-                goto out;
-        }
-        sh->mdata_spb = _gf_false;
-        if (nsources == 0) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "No self-heal needed for %s",
-                        local->loc.path);
-
-                afr_sh_metadata_finish (frame, this);
-                goto out;
-        }
-
-        source = afr_sh_select_source (sh->sources, priv->child_count);
-
-        if (source == -1) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "No active sources found.");
-
-                afr_sh_metadata_finish (frame, this);
-                goto out;
-        }
-
-        sh->source = source;
-
-        /* detect changes not visible through pending flags -- JIC */
-        for (i = 0; i < priv->child_count; i++) {
-                if (i == source || sh->child_errno[i])
-                        continue;
-
-                if (PERMISSION_DIFFERS (&sh->buf[i], &sh->buf[source]))
-                        sh->sources[i] = 0;
-
-                if (OWNERSHIP_DIFFERS (&sh->buf[i], &sh->buf[source]))
-                        sh->sources[i] = 0;
-        }
-
-        if ((!IA_ISREG (sh->buf[source].ia_type)) &&
-            (!IA_ISDIR (sh->buf[source].ia_type))) {
-                afr_reset_children (sh->fresh_children, priv->child_count);
-                afr_get_fresh_children (sh->success_children, sh->sources,
-                                        sh->fresh_children, priv->child_count);
-                afr_inode_set_read_ctx (this, sh->inode, sh->source,
-                                        sh->fresh_children);
-        }
-
-        afr_sh_metadata_sync_prepare (frame, this);
+	afr_private_t *priv = NULL;
+	int ret = -1;
+	unsigned char *sources = NULL;
+	unsigned char *sinks = NULL;
+	unsigned char *data_lock = NULL;
+	unsigned char *healed_sinks = NULL;
+	struct afr_reply *locked_replies = NULL;
+	int source = -1;
+
+	priv = this->private;
+
+	sources = alloca0 (priv->child_count);
+	sinks = alloca0 (priv->child_count);
+	healed_sinks = alloca0 (priv->child_count);
+	data_lock = alloca0 (priv->child_count);
+
+	locked_replies = alloca0 (sizeof (*locked_replies) * priv->child_count);
+
+	ret = afr_selfheal_inodelk (frame, this, inode, this->name,
+				    LLONG_MAX - 1, 0, data_lock);
+	{
+		if (ret < 2) {
+			ret = -ENOTCONN;
+			goto unlock;
+		}
+
+		ret = __afr_selfheal_metadata_prepare (frame, this, inode, data_lock,
+						       sources, sinks, healed_sinks,
+						       locked_replies);
+		if (ret < 0)
+			goto unlock;
+
+		source = ret;
+		ret = 0;
+	}
+unlock:
+	afr_selfheal_uninodelk (frame, this, inode, this->name,
+				LLONG_MAX -1, 0, data_lock);
+	if (ret < 0)
+		goto out;
+
+	ret = afr_selfheal_metadata_do (frame, this, inode, source, healed_sinks,
+					locked_replies);
+	if (ret)
+		goto out;
+
+	ret = afr_selfheal_undo_pending (frame, this, inode, sources, sinks,
+					 healed_sinks, AFR_METADATA_TRANSACTION,
+					 locked_replies, data_lock);
 out:
-        return;
-}
-
-int
-afr_sh_metadata_post_nonblocking_inodelk_cbk (call_frame_t *frame,
-                                              xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        if (int_lock->lock_op_ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG, "Non Blocking metadata "
-                        "inodelks failed for %s.", local->loc.path);
-                gf_log (this->name, GF_LOG_DEBUG, "Metadata self-heal "
-                        "failed for %s.", local->loc.path);
-                afr_sh_metadata_done (frame, this);
-        } else {
-
-                gf_log (this->name, GF_LOG_DEBUG, "Non Blocking metadata "
-                        "inodelks done for %s. Proceeding to FOP",
-                        local->loc.path);
-                afr_sh_common_lookup (frame, this, &local->loc,
-                                      afr_sh_metadata_fix, NULL,
-                                      AFR_LOOKUP_FAIL_CONFLICTS |
-                                      AFR_LOOKUP_FAIL_MISSING_GFIDS,
-                                      NULL);
-        }
-
-        return 0;
-}
-
-int
-afr_sh_metadata_lock (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        int_lock->transaction_lk_type = AFR_SELFHEAL_LK;
-        int_lock->selfheal_lk_type    = AFR_METADATA_SELF_HEAL_LK;
-
-        afr_set_lock_number (frame, this);
-
-        int_lock->lk_flock.l_start = LLONG_MAX - 1;
-        int_lock->lk_flock.l_len   = 0;
-        int_lock->lk_flock.l_type  = F_WRLCK;
-        int_lock->lock_cbk         = afr_sh_metadata_post_nonblocking_inodelk_cbk;
-
-        afr_nonblocking_inodelk (frame, this);
-
-        return 0;
+	return ret;
 }
 
 
 int
-afr_self_heal_metadata (call_frame_t *frame, xlator_t *this)
+afr_selfheal_metadata (call_frame_t *frame, xlator_t *this, inode_t *inode)
 {
-        afr_local_t   *local = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_private_t *priv = this->private;
-
-        local = frame->local;
-        sh = &local->self_heal;
-
-        /* Self-heal completion cbk changes inode split-brain status based on
-         * govinda_gOvinda, mdata_spb, data_spb values. Initialize mdata_spb
-         * with current split-brain status. If for some reason self-heal
-         * fails(locking phase etc), it makes sure we retain the split-brain
-         * status before this self-heal started.
-         */
-        sh->mdata_spb = afr_is_split_brain (this, sh->inode);
-
-        if (local->self_heal.do_metadata_self_heal && priv->metadata_self_heal) {
-                afr_sh_metadata_lock (frame, this);
-        } else {
-                afr_sh_metadata_done (frame, this);
-        }
-
-        return 0;
+	afr_private_t *priv = NULL;
+	unsigned char *locked_on = NULL;
+	int ret = 0;
+
+	priv = this->private;
+
+	locked_on = alloca0 (priv->child_count);
+
+	ret = afr_selfheal_tryinodelk (frame, this, inode, priv->sh_domain, 0, 0,
+				       locked_on);
+	{
+		if (ret < 2) {
+			/* Either less than two subvols available, or another
+			   selfheal (from another server) is in progress. Skip
+			   for now in any case there isn't anything to do.
+			*/
+			ret = -ENOTCONN;
+			goto unlock;
+		}
+
+		ret = __afr_selfheal_metadata (frame, this, inode, locked_on);
+	}
+unlock:
+	afr_selfheal_uninodelk (frame, this, inode, priv->sh_domain, 0, 0, locked_on);
+
+	return ret;
 }
diff --git a/xlators/cluster/afr/src/afr-self-heal-name.c b/xlators/cluster/afr/src/afr-self-heal-name.c
new file mode 100644
index 000000000..ce80b8da3
--- /dev/null
+++ b/xlators/cluster/afr/src/afr-self-heal-name.c
@@ -0,0 +1,457 @@
+/*
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "afr.h"
+#include "afr-self-heal.h"
+
+
+int
+__afr_selfheal_assign_gfid (call_frame_t *frame, xlator_t *this, inode_t *parent,
+			    uuid_t pargfid, const char *bname, inode_t *inode,
+			    struct afr_reply *replies, int gfid_idx)
+{
+	int i = 0;
+	afr_private_t *priv = NULL;
+	dict_t *xdata = NULL;
+	int ret = 0;
+	loc_t loc = {0, };
+
+	priv = this->private;
+
+	uuid_copy (parent->gfid, pargfid);
+
+	xdata = dict_new ();
+	if (!xdata) {
+		return -ENOMEM;
+	}
+
+	ret = dict_set_static_bin (xdata, "gfid-req",
+				   replies[gfid_idx].poststat.ia_gfid, 16);
+	if (ret) {
+		dict_destroy (xdata);
+		return -ENOMEM;
+	}
+
+	loc.parent = inode_ref (parent);
+	loc.inode = inode_ref (inode);
+	uuid_copy (loc.pargfid, pargfid);
+	loc.name = bname;
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (replies[i].op_ret == 0 || replies[i].op_errno != ENODATA)
+			continue;
+
+		ret = syncop_lookup (priv->children[i], &loc, xdata, 0, 0, 0);
+	}
+
+	loc_wipe (&loc);
+	dict_unref (xdata);
+
+	return ret;
+}
+
+
+int
+__afr_selfheal_name_impunge (call_frame_t *frame, xlator_t *this, inode_t *parent,
+			     uuid_t pargfid, const char *bname, inode_t *inode,
+			     struct afr_reply *replies, int gfid_idx)
+{
+	int i = 0;
+	afr_private_t *priv = NULL;
+	int ret = 0;
+
+	priv = this->private;
+
+	uuid_copy (parent->gfid, pargfid);
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!replies[i].valid)
+			continue;
+
+		if (uuid_compare (replies[i].poststat.ia_gfid,
+				  replies[gfid_idx].poststat.ia_gfid) == 0)
+			continue;
+
+		ret |= afr_selfheal_recreate_entry (frame, this, i, gfid_idx,
+						    parent, bname, inode, replies);
+	}
+
+	return ret;
+}
+
+
+int
+__afr_selfheal_name_expunge (call_frame_t *frame, xlator_t *this, inode_t *parent,
+			     uuid_t pargfid, const char *bname, inode_t *inode,
+			     struct afr_reply *replies)
+{
+	loc_t loc = {0, };
+	int i = 0;
+	afr_private_t *priv = NULL;
+	char g[64];
+	int ret = 0;
+
+	priv = this->private;
+
+	loc.parent = inode_ref (parent);
+	uuid_copy (loc.pargfid, pargfid);
+	loc.name = bname;
+	loc.inode = inode_ref (inode);
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!replies[i].valid)
+			continue;
+
+		if (replies[i].op_ret)
+			continue;
+
+		switch (replies[i].poststat.ia_type) {
+		case IA_IFDIR:
+			gf_log (this->name, GF_LOG_WARNING,
+				"expunging dir %s/%s (%s) on %s",
+				uuid_utoa (pargfid), bname,
+				uuid_utoa_r (replies[i].poststat.ia_gfid, g),
+				priv->children[i]->name);
+			ret |= syncop_rmdir (priv->children[i], &loc, 1);
+			break;
+		default:
+			gf_log (this->name, GF_LOG_WARNING,
+				"expunging file %s/%s (%s) on %s",
+				uuid_utoa (pargfid), bname,
+				uuid_utoa_r (replies[i].poststat.ia_gfid, g),
+				priv->children[i]->name);
+			ret |= syncop_unlink (priv->children[i], &loc);
+			break;
+		}
+	}
+
+	loc_wipe (&loc);
+
+	return ret;
+
+}
+
+
+int
+__afr_selfheal_name_do (call_frame_t *frame, xlator_t *this, inode_t *parent,
+			uuid_t pargfid, const char *bname, inode_t *inode,
+			unsigned char *sources, unsigned char *sinks,
+			unsigned char *healed_sinks, int source,
+			unsigned char *locked_on, struct afr_reply *replies)
+{
+	int i = 0;
+	afr_private_t *priv = NULL;
+	uuid_t gfid = {0, };
+	int gfid_idx = -1;
+	gf_boolean_t source_is_empty = _gf_true;
+	gf_boolean_t need_heal = _gf_false;
+	int first_idx = -1;
+	char g1[64],g2[64];
+
+	priv = this->private;
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!replies[i].valid)
+			continue;
+
+		if (first_idx == -1) {
+			first_idx = i;
+			continue;
+		}
+
+		if (replies[i].op_ret != replies[first_idx].op_ret)
+			need_heal = _gf_true;
+
+		if (uuid_compare (replies[i].poststat.ia_gfid,
+				  replies[first_idx].poststat.ia_gfid))
+			need_heal = _gf_true;
+	}
+
+	if (!need_heal)
+		return 0;
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!replies[i].valid)
+			continue;
+
+		if (!replies[i].op_ret && (source == -1 || sources[i])) {
+			source_is_empty = _gf_false;
+			break;
+		}
+	}
+
+	if (source_is_empty) {
+		return __afr_selfheal_name_expunge (frame, this, parent, pargfid,
+						    bname, inode, replies);
+	}
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!replies[i].valid)
+			continue;
+
+		if (uuid_is_null (replies[i].poststat.ia_gfid))
+			continue;
+
+		if (uuid_is_null (gfid)) {
+			uuid_copy (gfid, replies[i].poststat.ia_gfid);
+			gfid_idx = i;
+			continue;
+		}
+
+		if (sources[i] || source == -1) {
+			if (gfid_idx != -1 &&
+			    (sources[gfid_idx] || source == -1) &&
+			    uuid_compare (gfid, replies[i].poststat.ia_gfid)) {
+				gf_log (this->name, GF_LOG_WARNING,
+					"GFID mismatch for <gfid:%s>/%s "
+					"%s on %s and %s on %s",
+					uuid_utoa (pargfid), bname,
+					uuid_utoa_r (replies[i].poststat.ia_gfid, g1),
+					priv->children[i]->name,
+					uuid_utoa_r (replies[gfid_idx].poststat.ia_gfid, g2),
+					priv->children[gfid_idx]->name);
+				return -1;
+			}
+
+			uuid_copy (gfid, replies[i].poststat.ia_gfid);
+			gfid_idx = i;
+			continue;
+		}
+	}
+
+	if (gfid_idx == -1)
+		return -1;
+
+	__afr_selfheal_assign_gfid (frame, this, parent, pargfid, bname, inode,
+				    replies, gfid_idx);
+
+	return __afr_selfheal_name_impunge (frame, this, parent, pargfid,
+					    bname, inode, replies, gfid_idx);
+}
+
+
+int
+__afr_selfheal_name_finalize_source (xlator_t *this, unsigned char *sources,
+				     unsigned char *sinks, unsigned char *locked_on,
+				     struct afr_reply *replies)
+{
+	int i = 0;
+	afr_private_t *priv = NULL;
+	int source = -1;
+	int locked_count = 0;
+	int sources_count = 0;
+	int sinks_count = 0;
+
+	priv = this->private;
+
+	locked_count = AFR_COUNT (locked_on, priv->child_count);
+	sources_count = AFR_COUNT (sources, priv->child_count);
+	sinks_count = AFR_COUNT (sinks, priv->child_count);
+
+	if (locked_count == sinks_count || !sources_count) {
+		return -1;
+	}
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (sources[i]) {
+			source = i;
+			break;
+		}
+	}
+
+	return source;
+}
+
+
+int
+__afr_selfheal_name_prepare (call_frame_t *frame, xlator_t *this, inode_t *parent,
+			     uuid_t pargfid, unsigned char *locked_on,
+			     unsigned char *sources, unsigned char *sinks,
+			     unsigned char *healed_sinks, struct afr_reply *replies,
+			     int *source_p)
+{
+	int ret = -1;
+	int source = -1;
+	afr_private_t *priv = NULL;
+	int i = 0;
+
+	priv = this->private;
+
+	ret = afr_selfheal_unlocked_discover (frame, parent, pargfid, replies);
+	if (ret)
+		return ret;
+
+	ret = afr_selfheal_find_direction (frame, this, replies,
+					   AFR_ENTRY_TRANSACTION,
+					   locked_on, sources, sinks);
+	if (ret)
+		return ret;
+
+	source = __afr_selfheal_name_finalize_source (this, sources, sinks,
+						      locked_on, replies);
+	if (source < 0) {
+		/* If source is < 0 (typically split-brain), we perform a
+		   conservative merge of entries rather than erroring out */
+	}
+	*source_p = source;
+
+	for (i = 0; i < priv->child_count; i++)
+		/* Initialize the healed_sinks[] array optimistically to
+		   the intersection of to-be-healed (i.e sinks[]) and
+		   the list of servers which are up (i.e locked_on[]).
+
+		   As we encounter failures in the healing process, we
+		   will unmark the respective servers in the healed_sinks[]
+		   array.
+		*/
+		healed_sinks[i] = sinks[i] && locked_on[i];
+
+	return ret;
+}
+
+
+int
+afr_selfheal_name_do (call_frame_t *frame, xlator_t *this, inode_t *parent,
+		      uuid_t pargfid, const char *bname)
+{
+	afr_private_t *priv = NULL;
+	unsigned char *sources = NULL;
+	unsigned char *sinks = NULL;
+	unsigned char *healed_sinks = NULL;
+	unsigned char *locked_on = NULL;
+	int source = -1;
+	struct afr_reply *replies = NULL;
+	int ret = -1;
+	inode_t *inode = NULL;
+
+	priv = this->private;
+
+	locked_on = alloca0 (priv->child_count);
+	sources = alloca0 (priv->child_count);
+	sinks = alloca0 (priv->child_count);
+	healed_sinks = alloca0 (priv->child_count);
+
+	replies = alloca0 (priv->child_count * sizeof(*replies));
+
+	ret = afr_selfheal_entrylk (frame, this, parent, this->name, bname,
+				    locked_on);
+	{
+		if (ret < 2) {
+			ret = -ENOTCONN;
+			goto unlock;
+		}
+
+		ret = __afr_selfheal_name_prepare (frame, this, parent, pargfid,
+						   locked_on, sources, sinks,
+						   healed_sinks, replies,
+						   &source);
+		if (ret)
+			goto unlock;
+
+		inode = afr_selfheal_unlocked_lookup_on (frame, parent, bname,
+							 replies, locked_on);
+		if (!inode) {
+			ret = -ENOMEM;
+			goto unlock;
+		}
+
+		ret = __afr_selfheal_name_do (frame, this, parent, pargfid, bname,
+					      inode, sources, sinks, healed_sinks,
+					      source, locked_on, replies);
+	}
+unlock:
+	afr_selfheal_unentrylk (frame, this, parent, this->name, bname,
+				locked_on);
+	if (inode)
+		inode_unref (inode);
+
+	return ret;
+}
+
+
+int
+afr_selfheal_name_unlocked_inspect (call_frame_t *frame, xlator_t *this,
+				    inode_t *parent, uuid_t pargfid,
+				    const char *bname, gf_boolean_t *need_heal)
+{
+	afr_private_t *priv = NULL;
+	int i = 0;
+	struct afr_reply *replies = NULL;
+	inode_t *inode = NULL;
+	int first_idx = -1;
+
+	priv = this->private;
+
+	replies = alloca0 (sizeof (*replies) * priv->child_count);
+
+	inode = afr_selfheal_unlocked_lookup_on (frame, parent, bname,
+						 replies, priv->child_up);
+	if (!inode)
+		return -ENOMEM;
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!replies[i].valid)
+			continue;
+
+		if (first_idx == -1) {
+			first_idx = i;
+			continue;
+		}
+
+		if (replies[i].op_ret != replies[first_idx].op_ret)
+			*need_heal = _gf_true;
+
+		if (uuid_compare (replies[i].poststat.ia_gfid,
+				  replies[first_idx].poststat.ia_gfid))
+			*need_heal = _gf_true;
+	}
+
+	if (inode)
+		inode_unref (inode);
+	return 0;
+}
+
+int
+afr_selfheal_name (xlator_t *this, uuid_t pargfid, const char *bname)
+{
+	inode_t *parent = NULL;
+	call_frame_t *frame = NULL;
+	int ret = -1;
+	gf_boolean_t need_heal = _gf_false;
+
+	parent = afr_inode_find (this, pargfid);
+	if (!parent)
+		goto out;
+
+	frame = afr_frame_create (this);
+	if (!frame)
+		goto out;
+
+	ret = afr_selfheal_name_unlocked_inspect (frame, this, parent, pargfid,
+						  bname, &need_heal);
+	if (ret)
+		goto out;
+
+	if (need_heal)
+		afr_selfheal_name_do (frame, this, parent, pargfid, bname);
+out:
+	if (parent)
+		inode_unref (parent);
+	if (frame)
+		AFR_STACK_DESTROY (frame);
+
+	return ret;
+}
diff --git a/xlators/cluster/afr/src/afr-self-heal.h b/xlators/cluster/afr/src/afr-self-heal.h
index 627a2115a..a1b972ac3 100644
--- a/xlators/cluster/afr/src/afr-self-heal.h
+++ b/xlators/cluster/afr/src/afr-self-heal.h
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -8,42 +8,160 @@
   cases as published by the Free Software Foundation.
 */
 
-#ifndef __AFR_SELF_HEAL_H__
-#define __AFR_SELF_HEAL_H__
 
-#include <sys/stat.h>
+#ifndef _AFR_SELFHEAL_H
+#define _AFR_SELFHEAL_H
 
-#define FILETYPE_DIFFERS(buf1,buf2) ((buf1)->ia_type != (buf2)->ia_type)
-#define PERMISSION_DIFFERS(buf1,buf2) (st_mode_from_ia ((buf1)->ia_prot, (buf1)->ia_type) != st_mode_from_ia ((buf2)->ia_prot, (buf2)->ia_type))
-#define OWNERSHIP_DIFFERS(buf1,buf2) (((buf1)->ia_uid != (buf2)->ia_uid) || ((buf1)->ia_gid != (buf2)->ia_gid))
-#define SIZE_DIFFERS(buf1,buf2) ((buf1)->ia_size != (buf2)->ia_size)
 
-#define SIZE_GREATER(buf1,buf2) ((buf1)->ia_size > (buf2)->ia_size)
+/* Perform fop on all UP subvolumes and wait for all callbacks to return */
+
+#define AFR_ONALL(frame, rfn, fop, args ...) do {			\
+	afr_local_t *__local = frame->local;				\
+	afr_private_t *__priv = frame->this->private;			\
+	int __i = 0, __count = 0;					\
+									\
+	afr_replies_wipe (__local, __priv);				\
+									\
+	for (__i = 0; __i < __priv->child_count; __i++) {		\
+		if (!__priv->child_up[__i]) continue;			\
+		STACK_WIND_COOKIE (frame, rfn, (void *)(long) __i,	\
+				   __priv->children[__i],		\
+				   __priv->children[__i]->fops->fop, args); \
+		__count++;						\
+	}								\
+	syncbarrier_wait (&__local->barrier, __count);			\
+	} while (0)
+
+
+/* Perform fop on all subvolumes represented by list[] array and wait
+   for all callbacks to return */
+
+#define AFR_ONLIST(list, frame, rfn, fop, args ...) do {		\
+	afr_local_t *__local = frame->local;				\
+	afr_private_t *__priv = frame->this->private;			\
+	int __i = 0, __count = 0;					\
+									\
+	afr_replies_wipe (__local, __priv);				\
+									\
+	for (__i = 0; __i < __priv->child_count; __i++) {		\
+		if (!list[__i]) continue;				\
+		STACK_WIND_COOKIE (frame, rfn, (void *)(long) __i,	\
+				   __priv->children[__i],		\
+				   __priv->children[__i]->fops->fop, args); \
+		__count++;						\
+	}								\
+	syncbarrier_wait (&__local->barrier, __count);			\
+	} while (0)
+
+
+#define AFR_SEQ(frame, rfn, fop, args ...) do {				\
+	afr_local_t *__local = frame->local;				\
+	afr_private_t *__priv = frame->this->private;			\
+	int __i = 0;							\
+									\
+	afr_replies_wipe (__local, __priv);				\
+									\
+	for (__i = 0; __i < __priv->child_count; __i++) {		\
+		if (!__priv->child_up[__i]) continue;			\
+		STACK_WIND_COOKIE (frame, rfn, (void *)(long) __i,	\
+				   __priv->children[__i],		\
+				   __priv->children[__i]->fops->fop, args); \
+		syncbarrier_wait (&__local->barrier, 1);		\
+	}								\
+	} while (0)
+
+
+#define ALLOC_MATRIX(n, type) ({type **__ptr = NULL; \
+	int __i; \
+	__ptr = alloca0 (n * sizeof(type *)); \
+	for (__i = 0; __i < n; __i++) __ptr[__i] = alloca0 (n * sizeof(type)); \
+	__ptr;})
+
+
+#define IA_EQUAL(f,s,field) (memcmp (&(f.ia_##field), &(s.ia_##field), sizeof (s.ia_##field)) == 0)
+
+
+int
+afr_selfheal (xlator_t *this, uuid_t gfid);
+
+int
+afr_selfheal_name (xlator_t *this, uuid_t gfid, const char *name);
+
+int
+afr_selfheal_data (call_frame_t *frame, xlator_t *this, inode_t *inode);
 
 int
-afr_sh_has_metadata_pending (dict_t *xattr, xlator_t *this);
+afr_selfheal_metadata (call_frame_t *frame, xlator_t *this, inode_t *inode);
+
 int
-afr_sh_has_entry_pending (dict_t *xattr, xlator_t *this);
+afr_selfheal_entry (call_frame_t *frame, xlator_t *this, inode_t *inode);
+
+
 int
-afr_sh_has_data_pending (dict_t *xattr, xlator_t *this);
+afr_selfheal_inodelk (call_frame_t *frame, xlator_t *this, inode_t *inode,
+		      char *dom, off_t off, size_t size,
+		      unsigned char *locked_on);
 
 int
-afr_self_heal_entry (call_frame_t *frame, xlator_t *this);
+afr_selfheal_tryinodelk (call_frame_t *frame, xlator_t *this, inode_t *inode,
+			 char *dom, off_t off, size_t size,
+			 unsigned char *locked_on);
 
 int
-afr_self_heal_data (call_frame_t *frame, xlator_t *this);
+afr_selfheal_uninodelk (call_frame_t *frame, xlator_t *this, inode_t *inode,
+			char *dom, off_t off, size_t size,
+			const unsigned char *locked_on);
 
 int
-afr_self_heal_metadata (call_frame_t *frame, xlator_t *this);
+afr_selfheal_entrylk (call_frame_t *frame, xlator_t *this, inode_t *inode,
+		      char *dom, const char *name, unsigned char *locked_on);
 
 int
-afr_self_heal_get_source (xlator_t *this, afr_local_t *local, dict_t **xattr);
+afr_selfheal_tryentrylk (call_frame_t *frame, xlator_t *this, inode_t *inode,
+			 char *dom, const char *name, unsigned char *locked_on);
 
 int
-afr_self_heal (call_frame_t *frame, xlator_t *this, inode_t *inode);
+afr_selfheal_unentrylk (call_frame_t *frame, xlator_t *this, inode_t *inode,
+			char *dom, const char *name, unsigned char *locked_on);
 
 int
-afr_lookup_select_read_child_by_txn_type (xlator_t *this, afr_local_t *local,
-                                          dict_t **xattr,
-                                          afr_transaction_type txn_type);
-#endif /* __AFR_SELF_HEAL_H__ */
+afr_selfheal_unlocked_discover (call_frame_t *frame, inode_t *inode,
+				uuid_t gfid, struct afr_reply *replies);
+
+inode_t *
+afr_selfheal_unlocked_lookup_on (call_frame_t *frame, inode_t *parent,
+				 const char *name, struct afr_reply *replies,
+				 unsigned char *lookup_on);
+
+int
+afr_selfheal_find_direction (call_frame_t *frame, xlator_t *this,
+			     struct afr_reply *replies,
+			     afr_transaction_type type, unsigned char *locked_on,
+			     unsigned char *sources, unsigned char *sinks);
+
+int
+afr_selfheal_extract_xattr (xlator_t *this, struct afr_reply *replies,
+			    afr_transaction_type type, int *dirty, int **matrix);
+
+int
+afr_selfheal_undo_pending (call_frame_t *frame, xlator_t *this, inode_t *inode,
+			   unsigned char *sources, unsigned char *sinks,
+			   unsigned char *healed_sinks, afr_transaction_type type,
+			   struct afr_reply *replies, unsigned char *locked_on);
+
+int
+afr_selfheal_recreate_entry (call_frame_t *frame, xlator_t *this, int dst,
+			     int source, inode_t *dir, const char *name,
+			     inode_t *inode, struct afr_reply *replies);
+
+int
+afr_selfheal_post_op (call_frame_t *frame, xlator_t *this, inode_t *inode,
+		      int subvol, dict_t *xattr);
+
+call_frame_t *
+afr_frame_create (xlator_t *this);
+
+inode_t *
+afr_inode_find (xlator_t *this, uuid_t gfid);
+
+#endif /* !_AFR_SELFHEAL_H */
diff --git a/xlators/cluster/afr/src/afr-self-heald.c b/xlators/cluster/afr/src/afr-self-heald.c
index 5a2172ce9..4bfe909bc 100644
--- a/xlators/cluster/afr/src/afr-self-heald.c
+++ b/xlators/cluster/afr/src/afr-self-heald.c
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -8,1168 +8,1249 @@
   cases as published by the Free Software Foundation.
 */
 
+
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
 #endif
+
 #include "afr.h"
-#include "syncop.h"
+#include "afr-self-heal.h"
 #include "afr-self-heald.h"
-#include "afr-self-heal-common.h"
 #include "protocol-common.h"
-#include "event-history.h"
 
-#define AFR_POLL_TIMEOUT 600
+#define SHD_INODE_LRU_LIMIT          2048
+#define AFR_EH_HEALED_LIMIT          1024
+#define AFR_EH_HEAL_FAIL_LIMIT       1024
+#define AFR_EH_SPLIT_BRAIN_LIMIT     1024
+#define AFR_STATISTICS_HISTORY_SIZE    50
 
-typedef enum {
-        STOP_CRAWL_ON_SINGLE_SUBVOL = 1
-} afr_crawl_flags_t;
 
-typedef enum {
-        HEAL = 1,
-        INFO
-} shd_crawl_op;
+#define ASSERT_LOCAL(this, healer)				\
+	if (!afr_shd_is_subvol_local(this, healer->subvol)) {	\
+		healer->local = _gf_false;			\
+		if (safe_break (healer)) {			\
+			break;					\
+		} else {					\
+			continue;				\
+		}						\
+	} else {						\
+		healer->local = _gf_true;			\
+	}
 
-typedef struct shd_dump {
-        dict_t   *dict;
-        xlator_t *this;
-        int      child;
-} shd_dump_t;
 
-typedef struct shd_event_ {
-        int     child;
-        char    *path;
-} shd_event_t;
+#define NTH_INDEX_HEALER(this, n) &((((afr_private_t *)this->private))->shd.index_healers[n])
+#define NTH_FULL_HEALER(this, n) &((((afr_private_t *)this->private))->shd.full_healers[n])
 
-typedef struct shd_pos_ {
-        int     child;
-        xlator_t *this;
-        afr_child_pos_t pos;
-} shd_pos_t;
+int afr_shd_gfid_to_path (xlator_t *this, xlator_t *subvol, uuid_t gfid, char **path_p);
 
-typedef int
-(*afr_crawl_done_cbk_t)  (int ret, call_frame_t *sync_frame, void *crawl_data);
+char *
+afr_subvol_name (xlator_t *this, int subvol)
+{
+	afr_private_t *priv = NULL;
 
-void
-afr_start_crawl (xlator_t *this, int idx, afr_crawl_type_t crawl,
-                 process_entry_cbk_t process_entry, void *op_data,
-                 gf_boolean_t exclusive, int crawl_flags,
-                 afr_crawl_done_cbk_t crawl_done);
+	priv = this->private;
+	if (subvol < 0 || subvol > priv->child_count)
+		return NULL;
 
-static int
-_crawl_directory (fd_t *fd, loc_t *loc, afr_crawl_data_t *crawl_data);
+	return priv->children[subvol]->name;
+}
 
-int
-afr_syncop_find_child_position (void *data);
 
-static int
-_loc_assign_gfid_path (loc_t *loc)
+void
+afr_destroy_crawl_event_data (void *data)
 {
-        int  ret = -1;
-        char gfid_path[64] = {0};
-
-        if (loc->inode && !uuid_is_null (loc->inode->gfid)) {
-                ret = inode_path (loc->inode, NULL, (char**)&loc->path);
-        } else if (!uuid_is_null (loc->gfid)) {
-                snprintf (gfid_path, sizeof (gfid_path), "<gfid:%s>",
-                          uuid_utoa (loc->gfid));
-                loc->path = gf_strdup (gfid_path);
-                if (loc->path)
-                        ret = 0;
-        }
-        return ret;
+        return;
 }
 
+
 void
-shd_cleanup_event (void *event)
+afr_destroy_shd_event_data (void *data)
 {
-        shd_event_t *shd_event = event;
+	shd_event_t *shd_event = data;
+
+	if (!shd_event)
+		return;
+	GF_FREE (shd_event->path);
 
-        if (!shd_event)
-                goto out;
-        if (shd_event->path)
-                GF_FREE (shd_event->path);
-        GF_FREE (shd_event);
-out:
         return;
 }
 
-int
-afr_get_local_child (afr_self_heald_t *shd, unsigned int child_count)
-{
-        int i = 0;
-        int ret = -1;
-        for (i = 0; i < child_count; i++) {
-                if (shd->pos[i] == AFR_POS_LOCAL) {
-                        ret = i;
-                        break;
-                }
-        }
-        return ret;
-}
 
-static int
-_build_index_loc (xlator_t *this, loc_t *loc, char *name, loc_t *parent)
+gf_boolean_t
+afr_shd_is_subvol_local (xlator_t *this, int subvol)
 {
-        int             ret = 0;
+	char *pathinfo = NULL;
+	afr_private_t *priv = NULL;
+	dict_t *xattr = NULL;
+	int ret = 0;
+	gf_boolean_t is_local = _gf_false;
+	loc_t loc = {0, };
 
-        uuid_copy (loc->pargfid, parent->inode->gfid);
-        loc->path = "";
-        loc->name = name;
-        loc->parent = inode_ref (parent->inode);
-        if (!loc->parent) {
-                loc->path = NULL;
-                loc_wipe (loc);
-                ret = -1;
-        }
-        return ret;
-}
+	priv = this->private;
 
-int
-_add_path_to_dict (xlator_t *this, dict_t *output, int child, char *path,
-                   struct timeval *tv, gf_boolean_t dyn)
-{
-        //subkey not used for now
-        int             ret = -1;
-        uint64_t        count = 0;
-        char            key[256] = {0};
-        int             xl_id = 0;
+	loc.inode = this->itable->root;
+	uuid_copy (loc.gfid, loc.inode->gfid);
 
-        ret = dict_get_int32 (output, this->name, &xl_id);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "xl does not have id");
-                goto out;
-        }
+	ret = syncop_getxattr (priv->children[subvol], &loc, &xattr,
+			       GF_XATTR_PATHINFO_KEY);
+	if (ret)
+		return _gf_false;
+	if (!xattr)
+		return _gf_false;
 
-        snprintf (key, sizeof (key), "%d-%d-count", xl_id, child);
-        ret = dict_get_uint64 (output, key, &count);
+	ret = dict_get_str (xattr, GF_XATTR_PATHINFO_KEY, &pathinfo);
+	if (ret)
+		return _gf_false;
 
-        snprintf (key, sizeof (key), "%d-%d-%"PRIu64, xl_id, child, count);
-        if (dyn)
-                ret = dict_set_dynstr (output, key, path);
-        else
-                ret = dict_set_str (output, key, path);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "%s: Could not add to output",
-                        path);
-                goto out;
-        }
+	afr_local_pathinfo (pathinfo, &is_local);
 
-        if (!tv)
-                goto inc_count;
-        snprintf (key, sizeof (key), "%d-%d-%"PRIu64"-time", xl_id,
-                  child, count);
-        ret = dict_set_uint32 (output, key, tv->tv_sec);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "%s: Could not set time",
-                        path);
-                goto out;
-        }
+	gf_log (this->name, GF_LOG_DEBUG, "subvol %s is %slocal",
+		priv->children[subvol]->name, is_local? "" : "not ");
 
-inc_count:
-        snprintf (key, sizeof (key), "%d-%d-count", xl_id, child);
-        ret = dict_set_uint64 (output, key, count + 1);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Could not increment count");
-                goto out;
-        }
-        ret = 0;
-out:
-        return ret;
+	return is_local;
 }
 
+
 int
-_get_path_from_gfid_loc (xlator_t *this, xlator_t *readdir_xl, loc_t *child,
-                         char **fpath)
+__afr_shd_healer_wait (struct subvol_healer *healer)
 {
-        dict_t          *xattr = NULL;
-        char            *path = NULL;
-        int             ret = -1;
+	afr_private_t *priv = NULL;
+	struct timespec wait_till = {0, };
+	int ret = 0;
 
-        ret = syncop_getxattr (readdir_xl, child, &xattr,
-                               GFID_TO_PATH_KEY);
-        if (ret)
-                goto out;
-        ret = dict_get_str (xattr, GFID_TO_PATH_KEY, &path);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get path for "
-                        "gfid %s", uuid_utoa (child->gfid));
-                goto out;
-        }
-        path = gf_strdup (path);
-        if (!path) {
-                ret = -1;
-                goto out;
-        }
-        ret = 0;
-out:
-        if (!ret)
-                *fpath = path;
-        if (xattr)
-                dict_unref (xattr);
-        return ret;
-}
+	priv = healer->this->private;
 
-int
-_add_event_to_dict (circular_buffer_t *cb, void *data)
-{
-        int               ret = 0;
-        shd_dump_t        *dump_data = NULL;
-        shd_event_t       *shd_event = NULL;
+disabled_loop:
+	wait_till.tv_sec = time (NULL) + 60;
 
-        dump_data = data;
-        shd_event = cb->data;
-        if (shd_event->child != dump_data->child)
-                goto out;
-        ret = _add_path_to_dict (dump_data->this, dump_data->dict,
-                                 dump_data->child, shd_event->path, &cb->tv,
-                                 _gf_false);
-out:
-        return ret;
+	while (!healer->rerun) {
+		ret = pthread_cond_timedwait (&healer->cond,
+					      &healer->mutex,
+					      &wait_till);
+		if (ret == ETIMEDOUT)
+			break;
+	}
+
+	ret = healer->rerun;
+	healer->rerun = 0;
+
+	if (!priv->shd.enabled)
+		goto disabled_loop;
+
+	return ret;
 }
 
+
 int
-_add_eh_to_dict (xlator_t *this, eh_t *eh, dict_t *dict, int child)
+afr_shd_healer_wait (struct subvol_healer *healer)
 {
-        shd_dump_t dump_data = {0};
+	int ret = 0;
 
-        dump_data.this = this;
-        dump_data.dict = dict;
-        dump_data.child = child;
-        eh_dump (eh, &dump_data, _add_event_to_dict);
-        return 0;
+	pthread_mutex_lock (&healer->mutex);
+	{
+		ret = __afr_shd_healer_wait (healer);
+	}
+	pthread_mutex_unlock (&healer->mutex);
+
+	return ret;
 }
 
-int
-_add_summary_to_dict (xlator_t *this, afr_crawl_data_t *crawl_data,
-                      gf_dirent_t *entry,
-                      loc_t *childloc, loc_t *parentloc, struct iatt *iattr)
+
+gf_boolean_t
+safe_break (struct subvol_healer *healer)
 {
-        dict_t          *output = NULL;
-        xlator_t        *readdir_xl = NULL;
-        int             ret = -1;
-        char            *path = NULL;
+	gf_boolean_t ret = _gf_false;
 
-        if (uuid_is_null (childloc->gfid))
-                goto out;
+	pthread_mutex_lock (&healer->mutex);
+	{
+		if (healer->rerun)
+			goto unlock;
 
-        output = crawl_data->op_data;
-        readdir_xl = crawl_data->readdir_xl;
+		healer->running = _gf_false;
+		ret = _gf_true;
+	}
+unlock:
+	pthread_mutex_unlock (&healer->mutex);
+
+	return ret;
+}
 
-        ret = _get_path_from_gfid_loc (this, readdir_xl, childloc, &path);
-        if (ret)
-                goto out;
 
-        ret = _add_path_to_dict (this, output, crawl_data->child, path, NULL,
-                                 _gf_true);
+inode_t *
+afr_shd_inode_find (xlator_t *this, xlator_t *subvol, uuid_t gfid)
+{
+	inode_t *inode = NULL;
+	int ret = 0;
+	loc_t loc = {0, };
+	struct iatt iatt = {0, };
+
+	inode = inode_find (this->itable, gfid);
+	if (inode)
+		goto out;
+
+	loc.inode = inode_new (this->itable);
+	if (!loc.inode)
+		goto out;
+	uuid_copy (loc.gfid, gfid);
+
+	ret = syncop_lookup (subvol, &loc, NULL, &iatt, NULL, NULL);
+	if (ret < 0)
+		goto out;
+
+	inode = inode_link (loc.inode, NULL, NULL, &iatt);
+	if (inode)
+		inode_lookup (inode);
 out:
-        if (ret && path)
-                GF_FREE (path);
-        return ret;
+	loc_wipe (&loc);
+	return inode;
 }
 
-void
-_remove_stale_index (xlator_t *this, xlator_t *readdir_xl,
-                     loc_t *parent, char *fname)
-{
-        int              ret = 0;
-        loc_t            index_loc = {0};
 
-        ret = _build_index_loc (this, &index_loc, fname, parent);
-        if (ret)
-                goto out;
-        gf_log (this->name, GF_LOG_DEBUG, "Removing stale index "
-                "for %s on %s", index_loc.name, readdir_xl->name);
-        ret = syncop_unlink (readdir_xl, &index_loc);
-        if (ret && (errno != ENOENT)) {
-                gf_log (this->name, GF_LOG_ERROR, "%s: Failed to remove"
-                        " index on %s - %s", index_loc.name,
-                        readdir_xl->name, strerror (errno));
-        }
-        index_loc.path = NULL;
-        loc_wipe (&index_loc);
+fd_t *
+afr_shd_index_opendir (xlator_t *this, int child)
+{
+	fd_t *fd = NULL;
+	afr_private_t *priv = NULL;
+	xlator_t *subvol = NULL;
+	loc_t rootloc = {0, };
+	inode_t *inode = NULL;
+	int ret = 0;
+	dict_t *xattr = NULL;
+	void *index_gfid = NULL;
+
+	priv = this->private;
+	subvol = priv->children[child];
+
+	rootloc.inode = inode_ref (this->itable->root);
+	uuid_copy (rootloc.gfid, rootloc.inode->gfid);
+
+	ret = syncop_getxattr (subvol, &rootloc, &xattr,
+			       GF_XATTROP_INDEX_GFID);
+	if (ret || !xattr) {
+		errno = -ret;
+		goto out;
+	}
+
+	ret = dict_get_ptr (xattr, GF_XATTROP_INDEX_GFID, &index_gfid);
+	if (ret)
+		goto out;
+
+	gf_log (this->name, GF_LOG_DEBUG, "index-dir gfid for %s: %s",
+		subvol->name, uuid_utoa (index_gfid));
+
+	inode = afr_shd_inode_find (this, subvol, index_gfid);
+	if (!inode)
+		goto out;
+	fd = fd_anonymous (inode);
 out:
-        return;
+	loc_wipe (&rootloc);
+	if (xattr)
+		dict_unref (xattr);
+	return fd;
 }
 
-void
-_crawl_post_sh_action (xlator_t *this, loc_t *parent, loc_t *child,
-                       int32_t op_ret, int32_t op_errno, dict_t *xattr_rsp,
-                       afr_crawl_data_t *crawl_data)
+
+int
+afr_shd_index_purge (xlator_t *subvol, inode_t *inode, char *name)
 {
-        int              ret = 0;
-        afr_private_t    *priv = NULL;
-        afr_self_heald_t *shd = NULL;
-        eh_t             *eh = NULL;
-        char             *path = NULL;
-        shd_event_t      *event = NULL;
-        int32_t          sh_failed = 0;
-        gf_boolean_t     split_brain = 0;
-
-        priv = this->private;
-        shd  = &priv->shd;
-        if (crawl_data->crawl == INDEX) {
-                if ((op_ret < 0) && (op_errno == ENOENT)) {
-                        _remove_stale_index (this, crawl_data->readdir_xl,
-                                             parent, uuid_utoa (child->gfid));
-                        goto out;
-                }
-                ret = _get_path_from_gfid_loc (this, crawl_data->readdir_xl,
-                                               child, &path);
-                if (ret)
-                        goto out;
-        } else {
-                path = gf_strdup (child->path);
-                if (!path) {
-                        ret = -1;
-                        goto out;
-                }
-        }
+	loc_t loc = {0, };
+	int ret = 0;
 
-        if (xattr_rsp)
-                ret = dict_get_int32 (xattr_rsp, "sh-failed", &sh_failed);
-        split_brain = afr_is_split_brain (this, child->inode);
-        if ((op_ret < 0 && op_errno == EIO) || split_brain)
-                eh = shd->split_brain;
-        else if ((op_ret < 0) || sh_failed)
-                eh = shd->heal_failed;
-        else
-                eh = shd->healed;
-        ret = -1;
-        event = GF_CALLOC (1, sizeof (*event), gf_afr_mt_shd_event_t);
-        if (!event)
-                goto out;
-        event->child = crawl_data->child;
-        event->path = path;
-        ret = eh_save_history (eh, event);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "%s:Failed to save to "
-                        "event history, (%d, %s)", path, op_ret, strerror (op_errno));
-                goto out;
-        }
-        ret = 0;
-out:
-        if (ret && path)
-                GF_FREE (path);
-        return;
+	loc.parent = inode_ref (inode);
+	loc.name = name;
+
+	ret = syncop_unlink (subvol, &loc);
+
+	loc_wipe (&loc);
+	return ret;
 }
 
+
 int
-_self_heal_entry (xlator_t *this, afr_crawl_data_t *crawl_data, gf_dirent_t *entry,
-                  loc_t *child, loc_t *parent, struct iatt *iattr)
+afr_shd_selfheal_name (struct subvol_healer *healer, int child, uuid_t parent,
+		       const char *bname)
 {
-        struct iatt      parentbuf = {0};
-        int              ret = 0;
-        dict_t           *xattr_rsp = NULL;
+	int ret = -1;
 
-        gf_log (this->name, GF_LOG_DEBUG, "lookup %s", child->path);
+	ret = afr_selfheal_name (THIS, parent, bname);
 
-        ret = syncop_lookup (this, child, NULL,
-                             iattr, &xattr_rsp, &parentbuf);
-        _crawl_post_sh_action (this, parent, child, ret, errno, xattr_rsp,
-                               crawl_data);
-        if (xattr_rsp)
-                dict_unref (xattr_rsp);
-        return ret;
+	return ret;
 }
 
-static int
-afr_crawl_done  (int ret, call_frame_t *sync_frame, void *data)
+int
+afr_shd_selfheal (struct subvol_healer *healer, int child, uuid_t gfid)
 {
-        GF_FREE (data);
-        STACK_DESTROY (sync_frame->root);
-        return 0;
+	int ret = 0;
+	eh_t *eh = NULL;
+	afr_private_t *priv = NULL;
+	afr_self_heald_t *shd = NULL;
+	shd_event_t *shd_event = NULL;
+	char *path = NULL;
+	xlator_t *subvol = NULL;
+	xlator_t *this = NULL;
+	crawl_event_t *crawl_event = NULL;
+
+	this = healer->this;
+	priv = this->private;
+	shd = &priv->shd;
+	crawl_event = &healer->crawl_event;
+
+	subvol = priv->children[child];
+
+	ret = afr_selfheal (this, gfid);
+
+	if (ret == -EIO) {
+		eh = shd->split_brain;
+		crawl_event->split_brain_count++;
+	} else if (ret < 0) {
+		eh = shd->heal_failed;
+		crawl_event->heal_failed_count++;
+	} else if (ret == 0) {
+		eh = shd->healed;
+		crawl_event->healed_count++;
+	}
+
+	afr_shd_gfid_to_path (this, subvol, gfid, &path);
+	if (!path)
+		return ret;
+
+	if (eh) {
+		shd_event = GF_CALLOC (1, sizeof(*shd_event),
+				       gf_afr_mt_shd_event_t);
+		if (!shd_event) {
+			GF_FREE (path);
+			return ret;
+		}
+
+		shd_event->child = child;
+		shd_event->path = path;
+
+		if (eh_save_history (eh, shd_event) < 0) {
+			GF_FREE (shd_event);
+			GF_FREE (path);
+		}
+	}
+	return ret;
 }
 
+
 void
-_do_self_heal_on_subvol (xlator_t *this, int child, afr_crawl_type_t crawl)
+afr_shd_sweep_prepare (struct subvol_healer *healer)
 {
-        afr_start_crawl (this, child, crawl, _self_heal_entry,
-                         NULL, _gf_true, STOP_CRAWL_ON_SINGLE_SUBVOL,
-                         afr_crawl_done);
+	crawl_event_t *event = NULL;
+
+	event = &healer->crawl_event;
+
+	event->healed_count = 0;
+	event->split_brain_count = 0;
+	event->heal_failed_count = 0;
+
+	time (&event->start_time);
+	event->end_time = 0;
 }
 
-gf_boolean_t
-_crawl_proceed (xlator_t *this, int child, int crawl_flags, char **reason)
+
+void
+afr_shd_sweep_done (struct subvol_healer *healer)
 {
-        afr_private_t           *priv = NULL;
-        afr_self_heald_t        *shd = NULL;
-        gf_boolean_t            proceed = _gf_false;
-        char                    *msg = NULL;
-
-        priv = this->private;
-        shd  = &priv->shd;
-        if (!shd->enabled) {
-                msg = "Self-heal daemon is not enabled";
-                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
-                goto out;
-        }
-        if (!priv->child_up[child]) {
-                gf_log (this->name, GF_LOG_ERROR, "Stopping crawl for %s , "
-                        "subvol went down", priv->children[child]->name);
-                msg = "Brick is Not connected";
-                goto out;
-        }
+	crawl_event_t *event = NULL;
+	crawl_event_t *history = NULL;
+	afr_self_heald_t *shd = NULL;
 
-        if (crawl_flags & STOP_CRAWL_ON_SINGLE_SUBVOL) {
-                if (afr_up_children_count (priv->child_up,
-                                           priv->child_count) < 2) {
-                        gf_log (this->name, GF_LOG_ERROR, "Stopping crawl as "
-                                "< 2 children are up");
-                        msg = "< 2 bricks in replica are running";
-                        goto out;
-                }
-        }
-        proceed = _gf_true;
-out:
-        if (reason)
-                *reason = msg;
-        return proceed;
+	event = &healer->crawl_event;
+	shd = &(((afr_private_t *)healer->this->private)->shd);
+
+	time (&event->end_time);
+	history = memdup (event, sizeof (*event));
+	event->start_time = 0;
+
+	if (!history)
+		return;
+
+	if (eh_save_history (shd->statistics[healer->subvol], history) < 0)
+		GF_FREE (history);
 }
 
+
 int
-_do_crawl_op_on_local_subvols (xlator_t *this, afr_crawl_type_t crawl,
-                               shd_crawl_op op, dict_t *output)
+afr_shd_index_sweep (struct subvol_healer *healer)
 {
-        afr_private_t       *priv = NULL;
-        char                *status = NULL;
-        char                *subkey = NULL;
-        char                key[256] = {0};
-        shd_pos_t           pos_data = {0};
-        int                 op_ret = -1;
-        int                 xl_id = -1;
-        int                 i = 0;
-        int                 ret = 0;
-        int                 crawl_flags = 0;
-
-        priv = this->private;
-        if (op == HEAL)
-                crawl_flags |= STOP_CRAWL_ON_SINGLE_SUBVOL;
-
-        ret = dict_get_int32 (output, this->name, &xl_id);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Invalid input, "
-                        "translator-id is not available");
-                goto out;
-        }
-        pos_data.this = this;
-        subkey = "status";
-        for (i = 0; i < priv->child_count; i++) {
-                if (_crawl_proceed (this, i, crawl_flags, &status)) {
-                        pos_data.child = i;
-                        ret = synctask_new (this->ctx->env,
-                                            afr_syncop_find_child_position,
-                                            NULL, NULL, &pos_data);
-                        if (ret) {
-                                status = "Not able to find brick location";
-                        } else if (pos_data.pos == AFR_POS_REMOTE) {
-                                status = "brick is remote";
-                        } else {
-                                op_ret = 0;
-                                if (op == HEAL) {
-                                        status = "Started self-heal";
-                                        _do_self_heal_on_subvol (this, i,
-                                                                 crawl);
-                                } else {
-                                        status = "";
-                                        afr_start_crawl (this, i, INDEX,
-                                                         _add_summary_to_dict,
-                                                         output, _gf_false, 0,
-                                                         NULL);
-                                }
-                        }
-                        snprintf (key, sizeof (key), "%d-%d-%s", xl_id,
-                                  i, subkey);
-                        ret = dict_set_str (output, key, status);
-                        if (!op_ret && (crawl == FULL))
-                                break;
-                }
-                snprintf (key, sizeof (key), "%d-%d-%s", xl_id, i, subkey);
-                ret = dict_set_str (output, key, status);
-        }
-out:
-        return op_ret;
+	xlator_t *this = NULL;
+	int child = -1;
+	fd_t *fd = NULL;
+	xlator_t *subvol = NULL;
+	afr_private_t *priv = NULL;
+	off_t offset = 0;
+	gf_dirent_t entries;
+	gf_dirent_t *entry = NULL;
+	uuid_t gfid;
+	int ret = 0;
+	int count = 0;
+
+	this = healer->this;
+	child = healer->subvol;
+	priv = this->private;
+	subvol = priv->children[child];
+
+	fd = afr_shd_index_opendir (this, child);
+	if (!fd) {
+		gf_log (this->name, GF_LOG_WARNING,
+			"unable to opendir index-dir on %s", subvol->name);
+		return -errno;
+	}
+
+	INIT_LIST_HEAD (&entries.list);
+
+	while ((ret = syncop_readdir (subvol, fd, 131072, offset, &entries))) {
+		if (ret > 0)
+			ret = 0;
+		list_for_each_entry (entry, &entries.list, list) {
+			offset = entry->d_off;
+
+			if (!priv->shd.enabled) {
+				ret = -EBUSY;
+				break;
+			}
+
+			if (!strcmp (entry->d_name, ".") ||
+			    !strcmp (entry->d_name, ".."))
+				continue;
+
+			gf_log (this->name, GF_LOG_DEBUG, "got entry: %s",
+				entry->d_name);
+
+			ret = uuid_parse (entry->d_name, gfid);
+			if (ret)
+				continue;
+
+			ret = afr_shd_selfheal (healer, child, gfid);
+			if (ret == 0)
+				count++;
+
+			if (ret == -ENOENT || ret == -ESTALE) {
+				afr_shd_index_purge (subvol, fd->inode,
+						     entry->d_name);
+				ret = 0;
+			}
+		}
+
+		gf_dirent_free (&entries);
+		if (ret)
+			break;
+	}
+
+	if (fd)
+		fd_unref (fd);
+	if (!ret)
+		ret = count;
+	return ret;
 }
 
+
 int
-_do_self_heal_on_local_subvols (xlator_t *this, afr_crawl_type_t crawl,
-                                dict_t *output)
+afr_shd_full_sweep (struct subvol_healer *healer, inode_t *inode)
 {
-        return _do_crawl_op_on_local_subvols (this, crawl, HEAL, output);
+	fd_t *fd = NULL;
+	xlator_t *this = NULL;
+	xlator_t *subvol = NULL;
+	afr_private_t *priv = NULL;
+	off_t offset = 0;
+	gf_dirent_t entries;
+	gf_dirent_t *entry = NULL;
+	int ret = 0;
+
+	this = healer->this;
+	priv = this->private;
+	subvol = priv->children[healer->subvol];
+
+	fd = fd_anonymous (inode);
+	if (!fd)
+		return -errno;
+
+	INIT_LIST_HEAD (&entries.list);
+
+	while ((ret = syncop_readdirp (subvol, fd, 131072, offset, 0, &entries))) {
+		if (ret < 0)
+			break;
+
+		ret = gf_link_inodes_from_dirent (this, fd->inode, &entries);
+		if (ret)
+			break;
+
+		list_for_each_entry (entry, &entries.list, list) {
+			offset = entry->d_off;
+
+			if (!priv->shd.enabled) {
+				ret = -EBUSY;
+				break;
+			}
+
+			if (!strcmp (entry->d_name, ".") ||
+			    !strcmp (entry->d_name, ".."))
+				continue;
+
+			afr_shd_selfheal_name (healer, healer->subvol,
+					       inode->gfid, entry->d_name);
+
+			afr_shd_selfheal (healer, healer->subvol,
+					  entry->d_stat.ia_gfid);
+
+			if (entry->d_stat.ia_type == IA_IFDIR) {
+				ret = afr_shd_full_sweep (healer, entry->inode);
+				if (ret)
+					break;
+			}
+		}
+
+		gf_dirent_free (&entries);
+		if (ret)
+			break;
+	}
+
+	if (fd)
+		fd_unref (fd);
+	return ret;
 }
 
-int
-_get_index_summary_on_local_subvols (xlator_t *this, dict_t *output)
+
+void *
+afr_shd_index_healer (void *data)
 {
-        return _do_crawl_op_on_local_subvols (this, INDEX, INFO, output);
+	struct subvol_healer *healer = NULL;
+	xlator_t *this = NULL;
+	int ret = 0;
+
+	healer = data;
+	THIS = this = healer->this;
+
+	for (;;) {
+		afr_shd_healer_wait (healer);
+
+		ASSERT_LOCAL(this, healer);
+
+		do {
+			gf_log (this->name, GF_LOG_DEBUG,
+				"starting index sweep on subvol %s",
+				afr_subvol_name (this, healer->subvol));
+
+			afr_shd_sweep_prepare (healer);
+
+			ret = afr_shd_index_sweep (healer);
+
+			afr_shd_sweep_done (healer);
+			/*
+			  As long as at least one gfid was
+			  healed, keep retrying. We may have
+			  just healed a directory and thereby
+			  created entries for other gfids which
+			  could not be healed thus far.
+			*/
+
+			gf_log (this->name, GF_LOG_DEBUG,
+				"finished index sweep on subvol %s",
+				afr_subvol_name (this, healer->subvol));
+			/*
+			  Give a pause before retrying to avoid a busy loop
+			  in case the only entry in index is because of
+			  an ongoing I/O.
+			*/
+			sleep (1);
+		} while (ret > 0);
+	}
+
+	return NULL;
 }
 
-int
-_add_all_subvols_eh_to_dict (xlator_t *this, eh_t *eh, dict_t *dict)
+
+void *
+afr_shd_full_healer (void *data)
 {
-        afr_private_t           *priv = NULL;
-        afr_self_heald_t        *shd = NULL;
-        int                     i = 0;
+	struct subvol_healer *healer = NULL;
+	xlator_t *this = NULL;
+	int run = 0;
 
-        priv = this->private;
-        shd = &priv->shd;
+	healer = data;
+	THIS = this = healer->this;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (shd->pos[i] != AFR_POS_LOCAL)
-                        continue;
-                _add_eh_to_dict (this, eh, dict, i);
-        }
-        return 0;
-}
+	for (;;) {
+		pthread_mutex_lock (&healer->mutex);
+		{
+			run = __afr_shd_healer_wait (healer);
+			if (!run)
+				healer->running = _gf_false;
+		}
+		pthread_mutex_unlock (&healer->mutex);
 
-int
-afr_xl_op (xlator_t *this, dict_t *input, dict_t *output)
-{
-        gf_xl_afr_op_t   op = GF_AFR_OP_INVALID;
-        int              ret = 0;
-        afr_private_t    *priv = NULL;
-        afr_self_heald_t *shd = NULL;
-        int              xl_id = 0;
+		if (!run)
+			break;
 
-        priv = this->private;
-        shd = &priv->shd;
+		ASSERT_LOCAL(this, healer);
 
-        ret = dict_get_int32 (input, "xl-op", (int32_t*)&op);
-        if (ret)
-                goto out;
-        ret = dict_get_int32 (input, this->name, &xl_id);
-        if (ret)
-                goto out;
-        ret = dict_set_int32 (output, this->name, xl_id);
-        if (ret)
-                goto out;
-        switch (op) {
-        case GF_AFR_OP_HEAL_INDEX:
-                ret = _do_self_heal_on_local_subvols (this, INDEX, output);
-                break;
-        case GF_AFR_OP_HEAL_FULL:
-                ret = _do_self_heal_on_local_subvols (this, FULL, output);
-                break;
-        case GF_AFR_OP_INDEX_SUMMARY:
-                (void)_get_index_summary_on_local_subvols (this, output);
-                ret = 0;
-                break;
-        case GF_AFR_OP_HEALED_FILES:
-                ret = _add_all_subvols_eh_to_dict (this, shd->healed, output);
-                break;
-        case GF_AFR_OP_HEAL_FAILED_FILES:
-                ret = _add_all_subvols_eh_to_dict (this, shd->heal_failed,
-                                                   output);
-                break;
-        case GF_AFR_OP_SPLIT_BRAIN_FILES:
-                ret = _add_all_subvols_eh_to_dict (this, shd->split_brain,
-                                                   output);
-                break;
-        default:
-                gf_log (this->name, GF_LOG_ERROR, "Unknown set op %d", op);
-                break;
-        }
-out:
-        dict_del (output, this->name);
-        return ret;
-}
+		gf_log (this->name, GF_LOG_INFO,
+			"starting full sweep on subvol %s",
+			afr_subvol_name (this, healer->subvol));
 
-void
-afr_poll_self_heal (void *data)
-{
-        afr_private_t    *priv = NULL;
-        afr_self_heald_t *shd = NULL;
-        struct timeval   timeout = {0};
-        xlator_t         *this = NULL;
-        long             child = (long)data;
-        gf_timer_t       *old_timer = NULL;
-        gf_timer_t       *new_timer = NULL;
-        shd_pos_t        pos_data = {0};
-        int              ret = 0;
+		afr_shd_sweep_prepare (healer);
 
-        this = THIS;
-        priv = this->private;
-        shd = &priv->shd;
-
-        if (shd->pos[child] == AFR_POS_UNKNOWN) {
-                pos_data.this = this;
-                pos_data.child = child;
-                ret = synctask_new (this->ctx->env,
-                                    afr_syncop_find_child_position,
-                                    NULL, NULL, &pos_data);
-                if (!ret)
-                        shd->pos[child] = pos_data.pos;
-        }
-        if (shd->enabled && (shd->pos[child] == AFR_POS_LOCAL))
-                _do_self_heal_on_subvol (this, child, INDEX);
-        timeout.tv_sec = AFR_POLL_TIMEOUT;
-        timeout.tv_usec = 0;
-        //notify and previous timer should be synchronized.
-        LOCK (&priv->lock);
-        {
-                old_timer = shd->timer[child];
-                if (shd->pos[child] == AFR_POS_REMOTE)
-                        goto unlock;
-                shd->timer[child] = gf_timer_call_after (this->ctx, timeout,
-                                                         afr_poll_self_heal,
-                                                         data);
-                new_timer = shd->timer[child];
-        }
-unlock:
-        UNLOCK (&priv->lock);
-
-        if (old_timer)
-                gf_timer_call_cancel (this->ctx, old_timer);
-        if (!new_timer && (shd->pos[child] != AFR_POS_REMOTE)) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Could not create self-heal polling timer for %s",
-                        priv->children[child]->name);
-        }
-        return;
-}
+		afr_shd_full_sweep (healer, this->itable->root);
 
-static int
-afr_local_child_poll_self_heal  (int ret, call_frame_t *sync_frame, void *data)
-{
-        afr_self_heald_t *shd = NULL;
-        shd_pos_t        *pos_data = data;
-        afr_private_t    *priv = NULL;
+		afr_shd_sweep_done (healer);
 
-        if (ret)
-                goto out;
+		gf_log (this->name, GF_LOG_INFO,
+			"finished full sweep on subvol %s",
+			afr_subvol_name (this, healer->subvol));
+	}
 
-        priv = pos_data->this->private;
-        shd = &priv->shd;
-        shd->pos[pos_data->child] = pos_data->pos;
-        if (pos_data->pos != AFR_POS_REMOTE)
-                afr_poll_self_heal ((void*)(long)pos_data->child);
-out:
-        GF_FREE (data);
-        return 0;
+	return NULL;
 }
 
-void
-afr_proactive_self_heal (void *data)
+
+int
+afr_shd_healer_init (xlator_t *this, struct subvol_healer *healer)
 {
-        xlator_t         *this = NULL;
-        long             child = (long)data;
-        shd_pos_t        *pos_data = NULL;
-        int              ret = 0;
+	int ret = 0;
 
-        this = THIS;
+	ret = pthread_mutex_init (&healer->mutex, NULL);
+	if (ret)
+		goto out;
 
-        //Position of brick could have changed and it could be local now.
-        //Compute the position again
-        pos_data = GF_CALLOC (1, sizeof (*pos_data), gf_afr_mt_pos_data_t);
-        if (!pos_data)
-                goto out;
-        pos_data->this = this;
-        pos_data->child = child;
-        ret = synctask_new (this->ctx->env, afr_syncop_find_child_position,
-                            afr_local_child_poll_self_heal, NULL, pos_data);
-        if (ret)
-                goto out;
+	ret = pthread_cond_init (&healer->cond, NULL);
+	if (ret)
+		goto out;
+
+	healer->this = this;
+	healer->running = _gf_false;
+	healer->rerun = _gf_false;
+	healer->local = _gf_false;
 out:
-        return;
+	return ret;
 }
 
+
 int
-_link_inode_update_loc (xlator_t *this, loc_t *loc, struct iatt *iattr)
+afr_shd_healer_spawn (xlator_t *this, struct subvol_healer *healer,
+		      void *(threadfn)(void *))
 {
-        inode_t       *link_inode = NULL;
-        int           ret = -1;
+	int ret = 0;
+
+	pthread_mutex_lock (&healer->mutex);
+	{
+		if (healer->running) {
+			pthread_cond_signal (&healer->cond);
+		} else {
+			ret = gf_thread_create (&healer->thread, NULL,
+						threadfn, healer);
+			if (ret)
+				goto unlock;
+			healer->running = 1;
+		}
+
+		healer->rerun = 1;
+	}
+unlock:
+	pthread_mutex_unlock (&healer->mutex);
 
-        link_inode = inode_link (loc->inode, NULL, NULL, iattr);
-        if (link_inode == NULL) {
-                gf_log (this->name, GF_LOG_ERROR, "inode link failed "
-                        "on the inode (%s)", uuid_utoa (iattr->ia_gfid));
-                goto out;
-        }
-        inode_unref (loc->inode);
-        loc->inode = link_inode;
-        ret = 0;
-out:
-        return ret;
+	return ret;
 }
 
+
 int
-afr_crawl_build_start_loc (xlator_t *this, afr_crawl_data_t *crawl_data,
-                           loc_t *dirloc)
+afr_shd_full_healer_spawn (xlator_t *this, int subvol)
 {
-        afr_private_t *priv = NULL;
-        dict_t        *xattr = NULL;
-        void          *index_gfid = NULL;
-        loc_t         rootloc = {0};
-        struct iatt   iattr = {0};
-        struct iatt   parent = {0};
-        int           ret = 0;
-        xlator_t      *readdir_xl = crawl_data->readdir_xl;
-
-        priv = this->private;
-        if (crawl_data->crawl == FULL) {
-                afr_build_root_loc (this, dirloc);
-        } else {
-                afr_build_root_loc (this, &rootloc);
-                ret = syncop_getxattr (readdir_xl, &rootloc, &xattr,
-                                       GF_XATTROP_INDEX_GFID);
-                if (ret < 0)
-                        goto out;
-                ret = dict_get_ptr (xattr, GF_XATTROP_INDEX_GFID, &index_gfid);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to get index "
-                                "dir gfid on %s", readdir_xl->name);
-                        goto out;
-                }
-                if (!index_gfid) {
-                        gf_log (this->name, GF_LOG_ERROR, "index gfid empty "
-                                "on %s", readdir_xl->name);
-                        ret = -1;
-                        goto out;
-                }
-                uuid_copy (dirloc->gfid, index_gfid);
-                dirloc->path = "";
-                dirloc->inode = inode_new (priv->root_inode->table);
-                ret = syncop_lookup (readdir_xl, dirloc, NULL,
-                                     &iattr, NULL, &parent);
-                if (ret < 0) {
-                        if (errno != ENOENT) {
-                                gf_log (this->name, GF_LOG_ERROR, "lookup "
-                                        "failed on index dir on %s - (%s)",
-                                        readdir_xl->name, strerror (errno));
-                        }
-                        goto out;
-                }
-                ret = _link_inode_update_loc (this, dirloc, &iattr);
-                if (ret)
-                        goto out;
-        }
-        ret = 0;
-out:
-        if (xattr)
-                dict_unref (xattr);
-        loc_wipe (&rootloc);
-        return ret;
+	return afr_shd_healer_spawn (this, NTH_FULL_HEALER (this, subvol),
+				     afr_shd_full_healer);
 }
 
+
 int
-afr_crawl_opendir (xlator_t *this, afr_crawl_data_t *crawl_data, fd_t **dirfd,
-                   loc_t *dirloc)
+afr_shd_index_healer_spawn (xlator_t *this, int subvol)
 {
-        fd_t          *fd   = NULL;
-        int           ret = 0;
-
-        if (crawl_data->crawl == FULL) {
-                fd = fd_create (dirloc->inode, crawl_data->pid);
-                if (!fd) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to create fd for %s", dirloc->path);
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = syncop_opendir (crawl_data->readdir_xl, dirloc, fd);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "opendir failed on %s", dirloc->path);
-                        goto out;
-                }
-        } else {
-                fd = fd_anonymous (dirloc->inode);
-        }
-        ret = 0;
-out:
-        if (!ret)
-                *dirfd = fd;
-        return ret;
+	return afr_shd_healer_spawn (this, NTH_INDEX_HEALER (this, subvol),
+				     afr_shd_index_healer);
 }
 
-xlator_t*
-afr_crawl_readdir_xl_get (xlator_t *this, afr_crawl_data_t *crawl_data)
-{
-        afr_private_t *priv = this->private;
-
-        if (crawl_data->crawl == FULL) {
-                return this;
-        } else {
-                return priv->children[crawl_data->child];
-        }
-        return NULL;
-}
 
 int
-afr_crawl_build_child_loc (xlator_t *this, loc_t *child, loc_t *parent,
-                           gf_dirent_t *entry, afr_crawl_data_t *crawl_data)
+afr_shd_dict_add_crawl_event (xlator_t *this, dict_t *output,
+			      crawl_event_t *crawl_event)
 {
-        int           ret = -1;
-        afr_private_t *priv = NULL;
+        int             ret = 0;
+        uint64_t        count = 0;
+        char            key[256] = {0};
+        int             xl_id = 0;
+        uint64_t        healed_count = 0;
+        uint64_t        split_brain_count = 0;
+        uint64_t        heal_failed_count = 0;
+        char            *start_time_str = 0;
+        char            *end_time_str = NULL;
+        char            *crawl_type = NULL;
+        int             progress = -1;
+	int             child = -1;
 
-        priv = this->private;
-        if (crawl_data->crawl == FULL) {
-                ret = afr_build_child_loc (this, child, parent, entry->d_name);
-        } else {
-                child->inode = inode_new (priv->root_inode->table);
-                if (!child->inode)
-                        goto out;
-                uuid_parse (entry->d_name, child->gfid);
-                ret = _loc_assign_gfid_path (child);
-        }
-out:
-        return ret;
-}
+	child = crawl_event->child;
+        healed_count = crawl_event->healed_count;
+        split_brain_count = crawl_event->split_brain_count;
+        heal_failed_count = crawl_event->heal_failed_count;
+        crawl_type = crawl_event->crawl_type;
 
-static int
-_process_entries (xlator_t *this, loc_t *parentloc, gf_dirent_t *entries,
-                  off_t *offset, afr_crawl_data_t *crawl_data)
-{
-        gf_dirent_t      *entry = NULL;
-        gf_dirent_t      *tmp = NULL;
-        int              ret = 0;
-        loc_t            entry_loc = {0};
-        fd_t             *fd = NULL;
-        struct iatt      iattr = {0};
-
-        list_for_each_entry_safe (entry, tmp, &entries->list, list) {
-                if (!_crawl_proceed (this, crawl_data->child,
-                                     crawl_data->crawl_flags, NULL)) {
-                        ret = -1;
-                        goto out;
-                }
-                *offset = entry->d_off;
-                if (IS_ENTRY_CWD (entry->d_name) ||
-                    IS_ENTRY_PARENT (entry->d_name))
-                        continue;
-                if ((crawl_data->crawl == FULL) &&
-                     uuid_is_null (entry->d_stat.ia_gfid)) {
-                        gf_log (this->name, GF_LOG_WARNING, "%s/%s: No "
-                                "gfid present skipping",
-                                parentloc->path, entry->d_name);
-                        continue;
-                }
-
-                loc_wipe (&entry_loc);
-                ret = afr_crawl_build_child_loc (this, &entry_loc, parentloc,
-                                                 entry, crawl_data);
-                if (ret)
-                        goto out;
+	if (!crawl_event->start_time)
+		goto out;
 
-                ret = crawl_data->process_entry (this, crawl_data, entry,
-                                                 &entry_loc, parentloc, &iattr);
+        start_time_str = gf_strdup (ctime (&crawl_event->start_time));
 
-                if (ret)
-                        continue;
+	if (crawl_event->end_time)
+		end_time_str = gf_strdup (ctime (&crawl_event->end_time));
 
-                ret = _link_inode_update_loc (this, &entry_loc, &iattr);
-                if (ret)
-                        goto out;
-                if (crawl_data->crawl == INDEX)
-                        continue;
-
-                if (!IA_ISDIR (iattr.ia_type))
-                        continue;
-                fd = NULL;
-                ret = afr_crawl_opendir (this, crawl_data, &fd, &entry_loc);
-                if (ret)
-                        continue;
-                ret = _crawl_directory (fd, &entry_loc, crawl_data);
-                if (fd)
-                        fd_unref (fd);
+        ret = dict_get_int32 (output, this->name, &xl_id);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "xl does not have id");
+                goto out;
         }
-        ret = 0;
-out:
-        loc_wipe (&entry_loc);
-        return ret;
-}
 
-static int
-_crawl_directory (fd_t *fd, loc_t *loc, afr_crawl_data_t *crawl_data)
-{
-        xlator_t        *this = NULL;
-        off_t           offset   = 0;
-        gf_dirent_t     entries;
-        int             ret = 0;
-        gf_boolean_t    free_entries = _gf_false;
-        xlator_t        *readdir_xl = crawl_data->readdir_xl;
+        snprintf (key, sizeof (key), "statistics-%d-%d-count", xl_id, child);
+        ret = dict_get_uint64 (output, key, &count);
+
+
+        snprintf (key, sizeof (key), "statistics_healed_cnt-%d-%d-%"PRIu64,
+                  xl_id, child, count);
+        ret = dict_set_uint64(output, key, healed_count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+			"Could not add statistics_healed_count to outout");
+                goto out;
+	}
+
+        snprintf (key, sizeof (key), "statistics_sb_cnt-%d-%d-%"PRIu64,
+                  xl_id, child, count);
+        ret = dict_set_uint64 (output, key, split_brain_count);
+	if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+			"Could not add statistics_split_brain_count to outout");
+                goto out;
+        }
+
+        snprintf (key, sizeof (key), "statistics_crawl_type-%d-%d-%"PRIu64,
+                  xl_id, child, count);
+        ret = dict_set_str (output, key, crawl_type);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+			"Could not add statistics_crawl_type to output");
+                goto out;
+        }
 
-        INIT_LIST_HEAD (&entries.list);
-        this = THIS;
+        snprintf (key, sizeof (key), "statistics_heal_failed_cnt-%d-%d-%"PRIu64,
+                  xl_id, child, count);
+        ret = dict_set_uint64 (output, key, heal_failed_count);
+	if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+			"Could not add statistics_healed_failed_count to outout");
+                goto out;
+        }
 
-        GF_ASSERT (loc->inode);
+        snprintf (key, sizeof (key), "statistics_strt_time-%d-%d-%"PRIu64,
+                  xl_id, child, count);
+        ret = dict_set_dynstr (output, key, start_time_str);
+	if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+			"Could not add statistics_crawl_start_time to outout");
+                goto out;
+        } else {
+		start_time_str = NULL;
+	}
 
-        if (crawl_data->crawl == FULL)
-                gf_log (this->name, GF_LOG_DEBUG, "crawling %s", loc->path);
+	if (!end_time_str)
+                progress = 1;
         else
-                gf_log (this->name, GF_LOG_DEBUG, "crawling INDEX %s",
-                        uuid_utoa (loc->gfid));
-
-        while (1) {
-                if (crawl_data->crawl == FULL)
-                        ret = syncop_readdirp (readdir_xl, fd, 131072, offset,
-                                               NULL, &entries);
-                else
-                        ret = syncop_readdir (readdir_xl, fd, 131072, offset,
-                                              &entries);
-                if (ret <= 0)
-                        break;
-                ret = 0;
-                free_entries = _gf_true;
-
-                if (!_crawl_proceed (this, crawl_data->child,
-                                     crawl_data->crawl_flags, NULL)) {
-                        ret = -1;
-                        goto out;
-                }
-                if (list_empty (&entries.list))
-                        goto out;
+                progress = 0;
+
+        snprintf (key, sizeof (key), "statistics_end_time-%d-%d-%"PRIu64,
+                  xl_id, child, count);
+        if (!end_time_str)
+                end_time_str = gf_strdup ("Could not determine the end time");
+        ret = dict_set_dynstr (output, key, end_time_str);
+	if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+			"Could not add statistics_crawl_end_time to outout");
+                goto out;
+        } else {
+		end_time_str = NULL;
+	}
 
-                ret = _process_entries (this, loc, &entries, &offset,
-                                        crawl_data);
-                gf_dirent_free (&entries);
-                free_entries = _gf_false;
+        snprintf (key, sizeof (key), "statistics_inprogress-%d-%d-%"PRIu64,
+                  xl_id, child, count);
+
+        ret = dict_set_int32 (output, key, progress);
+	if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+			"Could not add statistics_inprogress to outout");
+                goto out;
         }
-        ret = 0;
+
+	snprintf (key, sizeof (key), "statistics-%d-%d-count", xl_id, child);
+	ret = dict_set_uint64 (output, key, count + 1);
+	if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+			"Could not increment the counter.");
+                goto out;
+	}
 out:
-        if (free_entries)
-                gf_dirent_free (&entries);
+	GF_FREE (start_time_str);
+	GF_FREE (end_time_str);
         return ret;
 }
 
-static char*
-position_str_get (afr_child_pos_t pos)
-{
-        switch (pos) {
-        case AFR_POS_UNKNOWN:
-                return "unknown";
-        case AFR_POS_LOCAL:
-                return "local";
-        case AFR_POS_REMOTE:
-                return "remote";
-        }
-        return NULL;
-}
 
 int
-afr_find_child_position (xlator_t *this, int child, afr_child_pos_t *pos)
+afr_shd_dict_add_path (xlator_t *this, dict_t *output, int child, char *path,
+		       struct timeval *tv)
 {
-        afr_private_t    *priv = NULL;
-        afr_self_heald_t *shd  = NULL;
-        dict_t           *xattr_rsp = NULL;
-        loc_t            loc = {0};
-        int              ret = 0;
-        char             *node_uuid = NULL;
+        int             ret = -1;
+        uint64_t        count = 0;
+        char            key[256] = {0};
+        int             xl_id = 0;
+
+        ret = dict_get_int32 (output, this->name, &xl_id);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "xl does not have id");
+                goto out;
+        }
 
-        priv = this->private;
-        shd  = &priv->shd;
+        snprintf (key, sizeof (key), "%d-%d-count", xl_id, child);
+        ret = dict_get_uint64 (output, key, &count);
 
-        afr_build_root_loc (this, &loc);
+        snprintf (key, sizeof (key), "%d-%d-%"PRIu64, xl_id, child, count);
+	ret = dict_set_dynstr (output, key, path);
 
-        ret = syncop_getxattr (priv->children[child], &loc, &xattr_rsp,
-                               GF_XATTR_NODE_UUID_KEY);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "getxattr failed on %s - "
-                        "(%s)", priv->children[child]->name, strerror (errno));
+                gf_log (this->name, GF_LOG_ERROR, "%s: Could not add to output",
+                        path);
                 goto out;
         }
 
-        ret = dict_get_str (xattr_rsp, GF_XATTR_NODE_UUID_KEY, &node_uuid);
+	if (tv) {
+		snprintf (key, sizeof (key), "%d-%d-%"PRIu64"-time", xl_id,
+			  child, count);
+		ret = dict_set_uint32 (output, key, tv->tv_sec);
+		if (ret) {
+			gf_log (this->name, GF_LOG_ERROR, "%s: Could not set time",
+				path);
+			goto out;
+		}
+	}
+
+        snprintf (key, sizeof (key), "%d-%d-count", xl_id, child);
+
+        ret = dict_set_uint64 (output, key, count + 1);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "node-uuid key not found on "
-                        "child %s", priv->children[child]->name);
+                gf_log (this->name, GF_LOG_ERROR, "Could not increment count");
                 goto out;
         }
 
-        if (!strcmp (node_uuid, shd->node_uuid))
-                *pos = AFR_POS_LOCAL;
-        else
-                *pos = AFR_POS_REMOTE;
-
-        gf_log (this->name, GF_LOG_DEBUG, "child %s is %s",
-                priv->children[child]->name, position_str_get (*pos));
+        ret = 0;
 out:
-        if (ret)
-                *pos = AFR_POS_UNKNOWN;
-        loc_wipe (&loc);
         return ret;
 }
 
+
 int
-afr_syncop_find_child_position (void *data)
+afr_shd_gfid_to_path (xlator_t *this, xlator_t *subvol, uuid_t gfid, char **path_p)
 {
-        shd_pos_t *pos_data = data;
-        int       ret = 0;
-
-        ret = afr_find_child_position (pos_data->this, pos_data->child,
-                                       &pos_data->pos);
-        return ret;
+	loc_t loc = {0,};
+	char *path = NULL;
+	dict_t *xattr = NULL;
+	int ret = 0;
+
+	uuid_copy (loc.gfid, gfid);
+	loc.inode = inode_new (this->itable);
+
+	ret = syncop_getxattr (subvol, &loc, &xattr, GFID_TO_PATH_KEY);
+	loc_wipe (&loc);
+	if (ret)
+		return ret;
+
+	ret = dict_get_str (xattr, GFID_TO_PATH_KEY, &path);
+	if (ret || !path)
+		return -EINVAL;
+
+	*path_p = gf_strdup (path);
+	if (!*path_p)
+		return -ENOMEM;
+	return 0;
 }
 
-static int
-afr_dir_crawl (void *data)
+
+int
+afr_shd_gather_index_entries (xlator_t *this, int child, dict_t *output)
 {
-        xlator_t            *this = NULL;
-        int                 ret = -1;
-        xlator_t            *readdir_xl = NULL;
-        fd_t                *fd = NULL;
-        loc_t               dirloc = {0};
-        afr_crawl_data_t    *crawl_data = data;
+	fd_t *fd = NULL;
+	xlator_t *subvol = NULL;
+	afr_private_t *priv = NULL;
+	off_t offset = 0;
+	gf_dirent_t entries;
+	gf_dirent_t *entry = NULL;
+	uuid_t gfid;
+	int ret = 0;
+	int count = 0;
+	char *path = NULL;
+
+	priv = this->private;
+	subvol = priv->children[child];
+
+	fd = afr_shd_index_opendir (this, child);
+	if (!fd) {
+		gf_log (this->name, GF_LOG_WARNING,
+			"unable to opendir index-dir on %s", subvol->name);
+		return -errno;
+	}
+
+	INIT_LIST_HEAD (&entries.list);
+
+	while ((ret = syncop_readdir (subvol, fd, 131072, offset, &entries))) {
+		if (ret > 0)
+			ret = 0;
+		list_for_each_entry (entry, &entries.list, list) {
+			offset = entry->d_off;
+
+			if (!strcmp (entry->d_name, ".") ||
+			    !strcmp (entry->d_name, ".."))
+				continue;
+
+			gf_log (this->name, GF_LOG_DEBUG, "got entry: %s",
+				entry->d_name);
+
+			ret = uuid_parse (entry->d_name, gfid);
+			if (ret)
+				continue;
+
+			path = NULL;
+			ret = afr_shd_gfid_to_path (this, subvol, gfid, &path);
+
+			if (ret == -ENOENT || ret == -ESTALE) {
+				afr_shd_index_purge (subvol, fd->inode,
+						     entry->d_name);
+				ret = 0;
+				continue;
+			}
+
+			ret = afr_shd_dict_add_path (this, output, child, path,
+						     NULL);
+		}
+
+		gf_dirent_free (&entries);
+		if (ret)
+			break;
+	}
+
+	if (fd)
+		fd_unref (fd);
+	if (!ret)
+		ret = count;
+	return ret;
+}
 
-        this = THIS;
 
-        if (!_crawl_proceed (this, crawl_data->child, crawl_data->crawl_flags,
-                             NULL))
-                goto out;
+int
+afr_add_shd_event (circular_buffer_t *cb, void *data)
+{
+	dict_t *output = NULL;
+	xlator_t *this = THIS;
+	afr_private_t *priv = NULL;
+	afr_self_heald_t *shd = NULL;
+	shd_event_t *shd_event = NULL;
+	char *path = NULL;
+
+	output = data;
+	priv = this->private;
+	shd = &priv->shd;
+	shd_event = cb->data;
+
+	if (!shd->index_healers[shd_event->child].local)
+		return 0;
+
+	path = gf_strdup (shd_event->path);
+	if (!path)
+		return -ENOMEM;
+
+	afr_shd_dict_add_path (this, output, shd_event->child, path,
+			       &cb->tv);
+	return 0;
+}
 
-        readdir_xl = afr_crawl_readdir_xl_get (this, crawl_data);
-        if (!readdir_xl)
-                goto out;
-        crawl_data->readdir_xl = readdir_xl;
+int
+afr_add_crawl_event (circular_buffer_t *cb, void *data)
+{
+	dict_t *output = NULL;
+	xlator_t *this = THIS;
+	afr_private_t *priv = NULL;
+	afr_self_heald_t *shd = NULL;
+	crawl_event_t *crawl_event = NULL;
 
-        ret = afr_crawl_build_start_loc (this, crawl_data, &dirloc);
-        if (ret)
-                goto out;
+	output = data;
+	priv = this->private;
+	shd = &priv->shd;
+	crawl_event = cb->data;
 
-        ret = afr_crawl_opendir (this, crawl_data, &fd, &dirloc);
-        if (ret)
-                goto out;
+	if (!shd->index_healers[crawl_event->child].local)
+		return 0;
 
-        ret = _crawl_directory (fd, &dirloc, crawl_data);
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR, "Crawl failed on %s",
-                        readdir_xl->name);
-        else
-                gf_log (this->name, GF_LOG_DEBUG, "Crawl completed "
-                        "on %s", readdir_xl->name);
-        if (crawl_data->crawl == INDEX)
-                dirloc.path = NULL;
-out:
-        if (fd)
-                fd_unref (fd);
-        if (crawl_data->crawl == INDEX)
-                dirloc.path = NULL;
-        loc_wipe (&dirloc);
-        return ret;
+	afr_shd_dict_add_crawl_event (this, output, crawl_event);
+
+	return 0;
 }
 
-static int
-afr_dir_exclusive_crawl (void *data)
-{
-        afr_private_t    *priv = NULL;
-        afr_self_heald_t *shd = NULL;
-        gf_boolean_t     crawl = _gf_false;
-        int              ret = 0;
-        int              child = -1;
-        xlator_t         *this = NULL;
-        afr_crawl_data_t *crawl_data = data;
-
-        this = THIS;
-        priv = this->private;
-        shd = &priv->shd;
-        child = crawl_data->child;
-
-        LOCK (&priv->lock);
-        {
-                if (shd->inprogress[child]) {
-                        if (shd->pending[child] != FULL)
-                                shd->pending[child] = crawl_data->crawl;
-                } else {
-                        shd->inprogress[child] = _gf_true;
-                        crawl = _gf_true;
-                }
-        }
-        UNLOCK (&priv->lock);
 
-        if (!crawl) {
-                gf_log (this->name, GF_LOG_INFO, "Another crawl is in progress "
-                        "for %s", priv->children[child]->name);
+int
+afr_selfheal_daemon_init (xlator_t *this)
+{
+	afr_private_t *priv = NULL;
+	afr_self_heald_t *shd = NULL;
+	int ret = -1;
+	int i = 0;
+
+	priv = this->private;
+	shd = &priv->shd;
+
+	this->itable = inode_table_new (SHD_INODE_LRU_LIMIT, this);
+	if (!this->itable)
+		goto out;
+
+	shd->index_healers = GF_CALLOC (sizeof(*shd->index_healers),
+					priv->child_count,
+					gf_afr_mt_subvol_healer_t);
+	if (!shd->index_healers)
+		goto out;
+
+	for (i = 0; i < priv->child_count; i++) {
+		shd->index_healers[i].subvol = i;
+		ret = afr_shd_healer_init (this, &shd->index_healers[i]);
+		if (ret)
+			goto out;
+	}
+
+	shd->full_healers = GF_CALLOC (sizeof(*shd->full_healers),
+				       priv->child_count,
+				       gf_afr_mt_subvol_healer_t);
+	if (!shd->full_healers)
+		goto out;
+	for (i = 0; i < priv->child_count; i++) {
+		shd->full_healers[i].subvol = i;
+		ret = afr_shd_healer_init (this, &shd->full_healers[i]);
+		if (ret)
+			goto out;
+	}
+
+	shd->healed = eh_new (AFR_EH_HEALED_LIMIT, _gf_false,
+			      afr_destroy_shd_event_data);
+        if (!shd->healed)
+		goto out;
+
+	shd->heal_failed = eh_new (AFR_EH_HEAL_FAIL_LIMIT, _gf_false,
+				   afr_destroy_shd_event_data);
+	if (!shd->heal_failed)
+		goto out;
+
+	shd->split_brain = eh_new (AFR_EH_SPLIT_BRAIN_LIMIT, _gf_false,
+				   afr_destroy_shd_event_data);
+	if (!shd->split_brain)
+		goto out;
+
+        shd->statistics = GF_CALLOC (sizeof(eh_t *), priv->child_count,
+				     gf_common_mt_eh_t);
+        if (!shd->statistics)
                 goto out;
+
+        for (i = 0; i < priv->child_count ; i++) {
+                shd->statistics[i] = eh_new (AFR_STATISTICS_HISTORY_SIZE,
+					     _gf_false,
+					     afr_destroy_crawl_event_data);
+                if (!shd->statistics[i])
+                        goto out;
+		shd->full_healers[i].crawl_event.child = i;
+		shd->full_healers[i].crawl_event.crawl_type = "FULL";
+		shd->index_healers[i].crawl_event.child = i;
+		shd->index_healers[i].crawl_event.crawl_type = "INDEX";
         }
 
-        do {
-                afr_dir_crawl (data);
-                LOCK (&priv->lock);
-                {
-                        if (shd->pending[child] != NONE) {
-                                crawl_data->crawl = shd->pending[child];
-                                shd->pending[child] = NONE;
-                        } else {
-                                shd->inprogress[child] = _gf_false;
-                                crawl = _gf_false;
-                        }
-                }
-                UNLOCK (&priv->lock);
-        } while (crawl);
+	ret = 0;
 out:
-        return ret;
+	return ret;
 }
 
-void
-afr_start_crawl (xlator_t *this, int idx, afr_crawl_type_t crawl,
-                 process_entry_cbk_t process_entry, void *op_data,
-                 gf_boolean_t exclusive, int crawl_flags,
-                 afr_crawl_done_cbk_t crawl_done)
-{
-        afr_private_t              *priv = NULL;
-        call_frame_t               *frame = NULL;
-        afr_crawl_data_t           *crawl_data = NULL;
-        int                        ret = 0;
-        int (*crawler) (void*) = NULL;
-
-        priv = this->private;
 
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame)
-                goto out;
+int
+afr_selfheal_childup (xlator_t *this, int subvol)
+{
+	afr_shd_index_healer_spawn (this, subvol);
 
-        afr_set_lk_owner (frame, this, frame->root);
-        afr_set_low_priority (frame);
-        crawl_data = GF_CALLOC (1, sizeof (*crawl_data),
-                                gf_afr_mt_crawl_data_t);
-        if (!crawl_data)
-                goto out;
-        crawl_data->process_entry = process_entry;
-        crawl_data->child = idx;
-        crawl_data->pid = frame->root->pid;
-        crawl_data->crawl = crawl;
-        crawl_data->op_data = op_data;
-        crawl_data->crawl_flags = crawl_flags;
-        gf_log (this->name, GF_LOG_DEBUG, "starting crawl %d for %s",
-                crawl_data->crawl, priv->children[idx]->name);
-
-        if (exclusive)
-                crawler = afr_dir_exclusive_crawl;
-        else
-                crawler = afr_dir_crawl;
-        ret = synctask_new (this->ctx->env, crawler,
-                            crawl_done, frame, crawl_data);
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR, "Could not create the "
-                        "task for %d ret %d", idx, ret);
-out:
-        return;
+	return 0;
 }
 
-void
-afr_build_root_loc (xlator_t *this, loc_t *loc)
-{
-        afr_private_t   *priv = NULL;
 
-        priv = this->private;
-        loc->path = gf_strdup ("/");
-        loc->name = "";
-        loc->inode = inode_ref (priv->root_inode);
-        uuid_copy (loc->gfid, loc->inode->gfid);
+int64_t
+afr_shd_get_index_count (xlator_t *this, int i)
+{
+	afr_private_t *priv = NULL;
+	xlator_t *subvol = NULL;
+	uint64_t count = 0;
+	loc_t rootloc = {0, };
+	dict_t *xattr = NULL;
+	int ret = -1;
+
+	priv = this->private;
+	subvol = priv->children[i];
+
+	rootloc.inode = inode_ref (this->itable->root);
+	uuid_copy (rootloc.gfid, rootloc.inode->gfid);
+
+	ret = syncop_getxattr (subvol, &rootloc, &xattr,
+			       GF_XATTROP_INDEX_COUNT);
+	loc_wipe (&rootloc);
+
+	if (ret < 0)
+		return -1;
+
+	ret = dict_get_uint64 (xattr, GF_XATTROP_INDEX_COUNT, &count);
+	if (ret)
+		return -1;
+	return count;
 }
 
+
 int
-afr_set_root_gfid (dict_t *dict)
+afr_xl_op (xlator_t *this, dict_t *input, dict_t *output)
 {
-        uuid_t gfid;
-        int ret = 0;
+        gf_xl_afr_op_t   op = GF_AFR_OP_INVALID;
+        int              ret = 0;
+        int              xl_id = 0;
+	afr_private_t   *priv = NULL;
+	afr_self_heald_t *shd = NULL;
+	struct subvol_healer *healer = NULL;
+	int i = 0;
+	char key[64];
+	int op_ret = 0;
+	int64_t cnt = 0;
 
-        memset (gfid, 0, 16);
-        gfid[15] = 1;
+	priv = this->private;
+	shd = &priv->shd;
 
-        ret = afr_set_dict_gfid (dict, gfid);
+	for (i = 0; i < priv->child_count; i++)
+		if (priv->child_up[i] == -1)
+			goto out;
 
-        return ret;
-}
+        ret = dict_get_int32 (input, "xl-op", (int32_t*)&op);
+        if (ret)
+                goto out;
+        ret = dict_get_int32 (input, this->name, &xl_id);
+        if (ret)
+                goto out;
+        ret = dict_set_int32 (output, this->name, xl_id);
+        if (ret)
+                goto out;
+        switch (op) {
+        case GF_AFR_OP_HEAL_INDEX:
+		op_ret = -1;
+
+		for (i = 0; i < priv->child_count; i++) {
+			healer = &shd->index_healers[i];
+			snprintf (key, 64, "%d-%d-status", xl_id, i);
+
+			if (!priv->child_up[i]) {
+				ret = dict_set_str (output, key,
+						    "Brick is not connected");
+			} else if (AFR_COUNT (priv->child_up,
+					      priv->child_count) < 2) {
+				ret = dict_set_str (output, key,
+						    "< 2 bricks in replica are up");
+			} else if (!afr_shd_is_subvol_local (this, healer->subvol)) {
+				ret = dict_set_str (output, key,
+						    "Brick is remote");
+			} else {
+				ret = dict_set_str (output, key,
+						    "Started self-heal");
+				afr_shd_index_healer_spawn (this, i);
+				op_ret = 0;
+			}
+		}
+                break;
+        case GF_AFR_OP_HEAL_FULL:
+		op_ret = -1;
+
+		for (i = 0; i < priv->child_count; i++) {
+			healer = &shd->full_healers[i];
+			snprintf (key, 64, "%d-%d-status", xl_id, i);
+
+			if (!priv->child_up[i]) {
+				ret = dict_set_str (output, key,
+						    "Brick is not connected");
+			} else if (AFR_COUNT (priv->child_up,
+					      priv->child_count) < 2) {
+				ret = dict_set_str (output, key,
+						    "< 2 bricks in replica are up");
+			} else if (!afr_shd_is_subvol_local (this, healer->subvol)) {
+				ret = dict_set_str (output, key,
+						    "Brick is remote");
+			} else {
+				ret = dict_set_str (output, key,
+						    "Started self-heal");
+				afr_shd_full_healer_spawn (this, i);
+				op_ret = 0;
+			}
+		}
+                break;
+        case GF_AFR_OP_INDEX_SUMMARY:
+		for (i = 0; i < priv->child_count; i++)
+			if (shd->index_healers[i].local)
+				afr_shd_gather_index_entries (this, i, output);
+                break;
+        case GF_AFR_OP_HEALED_FILES:
+		eh_dump (shd->healed, output, afr_add_shd_event);
+                break;
+        case GF_AFR_OP_HEAL_FAILED_FILES:
+		eh_dump (shd->heal_failed, output, afr_add_shd_event);
+                break;
+        case GF_AFR_OP_SPLIT_BRAIN_FILES:
+		eh_dump (shd->split_brain, output, afr_add_shd_event);
+                break;
+        case GF_AFR_OP_STATISTICS:
+		for (i = 0; i < priv->child_count; i++) {
+			eh_dump (shd->statistics[i], output,
+				 afr_add_crawl_event);
+			afr_shd_dict_add_crawl_event (this, output,
+						      &shd->index_healers[i].crawl_event);
+			afr_shd_dict_add_crawl_event (this, output,
+						      &shd->full_healers[i].crawl_event);
+		}
+                break;
+        case GF_AFR_OP_STATISTICS_HEAL_COUNT:
+        case GF_AFR_OP_STATISTICS_HEAL_COUNT_PER_REPLICA:
+		op_ret = -1;
+
+		for (i = 0; i < priv->child_count; i++) {
+			if (!priv->child_up[i]) {
+				snprintf (key, 64, "%d-%d-status", xl_id, i);
+				ret = dict_set_str (output, key,
+						    "Brick is not connected");
+			} else {
+				snprintf (key, 64, "%d-%d-hardlinks", xl_id, i);
+				cnt = afr_shd_get_index_count (this, i);
+				if (cnt >= 0) {
+					ret = dict_set_uint64 (output, key, cnt);
+				}
+				op_ret = 0;
+			}
+		}
+
+//                ret = _do_crawl_op_on_local_subvols (this, INDEX_TO_BE_HEALED,
+//                                                     STATISTICS_TO_BE_HEALED,
+//                                                     output);
+                break;
 
+        default:
+                gf_log (this->name, GF_LOG_ERROR, "Unknown set op %d", op);
+                break;
+        }
+out:
+        dict_del (output, this->name);
+        return op_ret;
+}
diff --git a/xlators/cluster/afr/src/afr-self-heald.h b/xlators/cluster/afr/src/afr-self-heald.h
index 8e6084596..10e229ee7 100644
--- a/xlators/cluster/afr/src/afr-self-heald.h
+++ b/xlators/cluster/afr/src/afr-self-heald.h
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -8,38 +8,65 @@
   cases as published by the Free Software Foundation.
 */
 
-#ifndef __AFR_SELF_HEALD_H__
-#define __AFR_SELF_HEALD_H__
-#include "xlator.h"
 
-#define IS_ROOT_PATH(path) (!strcmp (path, "/"))
-#define IS_ENTRY_CWD(entry) (!strcmp (entry, "."))
-#define IS_ENTRY_PARENT(entry) (!strcmp (entry, ".."))
-#define AFR_ALL_CHILDREN -1
+#ifndef _AFR_SELF_HEALD_H
+#define _AFR_SELF_HEALD_H
 
-typedef struct afr_crawl_data_ {
-        int                 child;
-        pid_t               pid;
-        afr_crawl_type_t    crawl;
-        xlator_t            *readdir_xl;
-        void                *op_data;
-        int                 crawl_flags;
-        int (*process_entry) (xlator_t *this, struct afr_crawl_data_ *crawl_data,
-                              gf_dirent_t *entry, loc_t *child, loc_t *parent,
-                              struct iatt *iattr);
-} afr_crawl_data_t;
+#include <pthread.h>
 
-typedef int (*process_entry_cbk_t) (xlator_t *this, afr_crawl_data_t *crawl_data,
-                              gf_dirent_t *entry, loc_t *child, loc_t *parent,
-                              struct iatt *iattr);
 
-void afr_build_root_loc (xlator_t *this, loc_t *loc);
+typedef struct {
+	int child;
+	char *path;
+} shd_event_t;
 
-int afr_set_root_gfid (dict_t *dict);
+typedef struct {
+	int      child;
+	uint64_t healed_count;
+        uint64_t split_brain_count;
+        uint64_t heal_failed_count;
+
+	/* If start_time is 0, it means crawler is not in progress
+	   and stats are not valid */
+	time_t   start_time;
+	/* If start_time is NOT 0 and end_time is 0, it means
+	   cralwer is in progress */
+        time_t   end_time;
+        char     *crawl_type;
+} crawl_event_t;
+
+struct subvol_healer {
+	xlator_t        *this;
+	int              subvol;
+	gf_boolean_t     local;
+	gf_boolean_t     running;
+	gf_boolean_t     rerun;
+	crawl_event_t    crawl_event;
+	pthread_mutex_t  mutex;
+	pthread_cond_t   cond;
+	pthread_t        thread;
+};
+
+typedef struct {
+	gf_boolean_t            iamshd;
+	gf_boolean_t            enabled;
+	struct subvol_healer   *index_healers;
+	struct subvol_healer   *full_healers;
+
+	eh_t                    *healed;
+        eh_t                    *heal_failed;
+        eh_t                    *split_brain;
+        eh_t                    **statistics;
+} afr_self_heald_t;
 
-void
-afr_proactive_self_heal (void *data);
+
+int
+afr_selfheal_childup (xlator_t *this, int subvol);
+
+int
+afr_selfheal_daemon_init (xlator_t *this);
 
 int
 afr_xl_op (xlator_t *this, dict_t *input, dict_t *output);
-#endif /* __AFR_SELF_HEALD_H__ */
+
+#endif /* !_AFR_SELF_HEALD_H */
diff --git a/xlators/cluster/afr/src/afr-transaction.c b/xlators/cluster/afr/src/afr-transaction.c
index b21e14a4b..205ff759e 100644
--- a/xlators/cluster/afr/src/afr-transaction.c
+++ b/xlators/cluster/afr/src/afr-transaction.c
@@ -11,196 +11,163 @@
 #include "dict.h"
 #include "byte-order.h"
 #include "common-utils.h"
+#include "timer.h"
 
 #include "afr.h"
 #include "afr-transaction.h"
 
 #include <signal.h>
 
+gf_boolean_t
+afr_changelog_pre_op_uninherit (call_frame_t *frame, xlator_t *this);
 
-#define LOCKED_NO       0x0        /* no lock held */
-#define LOCKED_YES      0x1        /* for DATA, METADATA, ENTRY and higher_path
-                                      of RENAME */
-#define LOCKED_LOWER    0x2        /* for lower_path of RENAME */
+gf_boolean_t
+afr_changelog_pre_op_update (call_frame_t *frame, xlator_t *this);
+
+int
+afr_changelog_do (call_frame_t *frame, xlator_t *this, dict_t *xattr,
+		  afr_changelog_resume_t changelog_resume);
 
 
-afr_fd_ctx_t *
-__afr_fd_ctx_get (fd_t *fd, xlator_t *this)
+int
+__afr_txn_write_fop (call_frame_t *frame, xlator_t *this)
 {
-        uint64_t       ctx = 0;
-        int            ret = 0;
-        afr_fd_ctx_t  *fd_ctx = NULL;
-        int            i = 0;
+        afr_local_t *local = NULL;
         afr_private_t *priv = NULL;
+        int call_count = -1;
+        int i = 0;
 
+        local = frame->local;
         priv = this->private;
 
-        ret = __fd_ctx_get (fd, this, &ctx);
-
-        if (ret < 0 && fd_is_anonymous (fd)) {
-                ret = __afr_fd_ctx_set (this, fd);
-                if (ret < 0)
-                        goto out;
-
-                ret = __fd_ctx_get (fd, this, &ctx);
-                if (ret < 0)
-                        goto out;
+        call_count = AFR_COUNT (local->transaction.pre_op, priv->child_count);
 
-                fd_ctx = (afr_fd_ctx_t *)(long) ctx;
-                for (i = 0; i < priv->child_count; i++)
-                        fd_ctx->opened_on[i] = AFR_FD_OPENED;
+        if (call_count == 0) {
+                local->transaction.resume (frame, this);
+                return 0;
         }
 
-        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
-out:
-        return fd_ctx;
-}
-
+        local->call_count = call_count;
 
-afr_fd_ctx_t *
-afr_fd_ctx_get (fd_t *fd, xlator_t *this)
-{
-        afr_fd_ctx_t  *fd_ctx = NULL;
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->transaction.pre_op[i]) {
+			local->transaction.wind (frame, this, i);
 
-        LOCK(&fd->lock);
-        {
-                fd_ctx = __afr_fd_ctx_get (fd, this);
+                        if (!--call_count)
+                                break;
+                }
         }
-        UNLOCK(&fd->lock);
 
-        return fd_ctx;
+        return 0;
 }
 
 
-static void
-afr_pid_save (call_frame_t *frame)
+int
+__afr_txn_write_done (call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t * local = NULL;
+        afr_local_t *local = NULL;
 
         local = frame->local;
 
-        local->saved_pid = frame->root->pid;
-}
-
-
-static void
-afr_pid_restore (call_frame_t *frame)
-{
-        afr_local_t * local = NULL;
+        local->transaction.unwind (frame, this);
 
-        local = frame->local;
+        AFR_STACK_DESTROY (frame);
 
-        frame->root->pid = local->saved_pid;
+        return 0;
 }
 
 
-static void
-__mark_all_pending (int32_t *pending[], int child_count,
-                    afr_transaction_type type)
+call_frame_t*
+afr_transaction_detach_fop_frame (call_frame_t *frame)
 {
-        int i = 0;
-        int j = 0;
-
-        for (i = 0; i < child_count; i++) {
-                j = afr_index_for_transaction_type (type);
-                pending[i][j] = hton32 (1);
-        }
-}
-
+        afr_local_t *   local = NULL;
+        call_frame_t   *fop_frame = NULL;
 
-static void
-__mark_child_dead (int32_t *pending[], int child_count, int child,
-                   afr_transaction_type type)
-{
-        int j = 0;
+        local = frame->local;
 
-        j = afr_index_for_transaction_type (type);
+        LOCK (&frame->lock);
+        {
+                fop_frame = local->transaction.main_frame;
+                local->transaction.main_frame = NULL;
+        }
+        UNLOCK (&frame->lock);
 
-        pending[child][j] = 0;
+        return fop_frame;
 }
 
 
 static void
-__mark_pre_op_done_on_fd (call_frame_t *frame, xlator_t *this, int child_index)
+afr_save_lk_owner (call_frame_t *frame)
 {
-        afr_local_t   *local = NULL;
-        afr_fd_ctx_t  *fd_ctx = NULL;
+        afr_local_t * local = NULL;
 
         local = frame->local;
 
-        if (!local->fd)
-                return;
-
-        fd_ctx = afr_fd_ctx_get (local->fd, this);
-
-        if (!fd_ctx)
-                goto out;
-
-        LOCK (&local->fd->lock);
-        {
-                if (local->transaction.type == AFR_DATA_TRANSACTION)
-                        fd_ctx->pre_op_done[child_index]++;
-        }
-        UNLOCK (&local->fd->lock);
-out:
-        return;
+        local->saved_lk_owner = frame->root->lk_owner;
 }
 
 
 static void
-__mark_pre_op_undone_on_fd (call_frame_t *frame, xlator_t *this, int child_index)
+afr_restore_lk_owner (call_frame_t *frame)
 {
-        afr_local_t   *local = NULL;
-        afr_fd_ctx_t  *fd_ctx = NULL;
+        afr_local_t * local = NULL;
 
         local = frame->local;
 
-        if (!local->fd)
-                return;
-
-        fd_ctx = afr_fd_ctx_get (local->fd, this);
-
-        if (!fd_ctx)
-                goto out;
-
-        LOCK (&local->fd->lock);
-        {
-                if (local->transaction.type == AFR_DATA_TRANSACTION)
-                        fd_ctx->pre_op_done[child_index]--;
-        }
-        UNLOCK (&local->fd->lock);
-out:
-        return;
+        frame->root->lk_owner = local->saved_lk_owner;
 }
 
-
-static void
-__mark_non_participant_children (int32_t *pending[], int child_count,
-                                 unsigned char *participants,
-                                 afr_transaction_type type)
+void
+__mark_all_success (call_frame_t *frame, xlator_t *this)
 {
-        int i = 0;
-        int j = 0;
+	afr_private_t *priv = NULL;
+	afr_local_t *local = NULL;
+	int i;
 
-        j = afr_index_for_transaction_type (type);
-        for (i = 0; i < child_count; i++) {
-                if (!participants[i])
-                        pending[i][j] = 0;
-        }
+	local = frame->local;
+	priv = this->private;
+
+	for (i = 0; i < priv->child_count; i++) {
+		local->transaction.failed_subvols[i] = 0;
+	}
 }
 
 
-static void
-__mark_all_success (int32_t *pending[], int child_count,
-                    afr_transaction_type type)
+int
+afr_transaction_perform_fop (call_frame_t *frame, xlator_t *this)
 {
-        int i;
-        int j;
+        afr_local_t     *local = NULL;
+        fd_t            *fd   = NULL;
 
-        for (i = 0; i < child_count; i++) {
-                j = afr_index_for_transaction_type (type);
-                pending[i][j] = hton32 (-1);
-        }
+        local = frame->local;
+        fd    = local->fd;
+
+        /*  Perform fops with the lk-owner from top xlator.
+         *  Eg: lk-owner of posix-lk and flush should be same,
+         *  flush cant clear the  posix-lks without that lk-owner.
+         */
+        afr_save_lk_owner (frame);
+        frame->root->lk_owner =
+                local->transaction.main_frame->root->lk_owner;
+
+	if (local->pre_op_compat)
+		/* old mode, pre-op was done as afr_changelog_do()
+		   just now, before OP */
+		afr_changelog_pre_op_update (frame, this);
+
+        /* The wake up needs to happen independent of
+           what type of fop arrives here. If it was
+           a write, then it has already inherited the
+           lock and changelog. If it was not a write,
+           then the presumption of the optimization (of
+           optimizing for successive write operations)
+           fails.
+        */
+        if (fd)
+                afr_delayed_changelog_wake_up (this, fd);
+        local->transaction.fop (frame, this);
+
+	return 0;
 }
 
 
@@ -266,64 +233,31 @@ __fop_changelog_needed (call_frame_t *frame, xlator_t *this)
         return op_ret;
 }
 
+
 int
-afr_set_pending_dict (afr_private_t *priv, dict_t *xattr, int32_t **pending)
+afr_set_pending_dict (afr_private_t *priv, dict_t *xattr, int **pending)
 {
         int i = 0;
         int ret = 0;
+	int pending_zero[AFR_NUM_CHANGE_LOGS] = {0, };
 
         for (i = 0; i < priv->child_count; i++) {
+		if (!memcmp (pending_zero, pending[i], sizeof (pending_zero)))
+			/* don't set xattrs for non-pending servers */
+			continue;
+
                 ret = dict_set_static_bin (xattr, priv->pending_key[i],
-                                           pending[i],
-                                AFR_NUM_CHANGE_LOGS * sizeof (int32_t));
+					   pending[i],
+					   AFR_NUM_CHANGE_LOGS * sizeof (int));
                 /* 3 = data+metadata+entry */
 
-                if (ret < 0)
-                        goto out;
-        }
-
-out:
-        return ret;
-}
-
-
-static int
-afr_set_piggyback_dict (afr_private_t *priv, dict_t *xattr, int32_t **pending,
-                        afr_transaction_type type)
-{
-        int i = 0;
-        int ret = 0;
-        int *arr = NULL;
-        int index = 0;
-        size_t pending_xattr_size = 3 * sizeof (int32_t);
-        /* 3 = data+metadata+entry */
-
-        index = afr_index_for_transaction_type (type);
-
-        for (i = 0; i < priv->child_count; i++) {
-                arr = GF_CALLOC (1, pending_xattr_size,
-                                 gf_afr_mt_char);
-                if (!arr) {
-                        ret = -1;
-                        goto out;
-                }
-
-                memcpy (arr, pending[i], pending_xattr_size);
-
-                arr[index] = hton32 (ntoh32(arr[index]) + 1);
-
-                ret = dict_set_bin (xattr, priv->pending_key[i],
-                                    arr, pending_xattr_size);
-
-                if (ret < 0)
-                        goto out;
+                if (ret)
+                        break;
         }
 
-out:
         return ret;
 }
 
-
 int
 afr_lock_server_count (afr_private_t *priv, afr_transaction_type type)
 {
@@ -349,432 +283,532 @@ afr_lock_server_count (afr_private_t *priv, afr_transaction_type type)
 
 /* {{{ pending */
 
-int32_t
-afr_changelog_post_op_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno, dict_t *xattr,
-                           dict_t *xdata)
+
+int
+afr_changelog_post_op_done (call_frame_t *frame, xlator_t *this)
 {
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
         afr_internal_lock_t *int_lock = NULL;
-        afr_private_t       *priv     = NULL;
-        afr_local_t         *local    = NULL;
-        int                  call_count = -1;
 
-        priv     = this->private;
-        local    = frame->local;
+	local = frame->local;
+	priv = this->private;
         int_lock = &local->internal_lock;
 
-        LOCK (&frame->lock);
-        {
-                call_count = --local->call_count;
-        }
-        UNLOCK (&frame->lock);
+	if (local->transaction.resume_stub) {
+		call_resume (local->transaction.resume_stub);
+		local->transaction.resume_stub = NULL;
+	}
 
-        if (call_count == 0) {
-                if (afr_lock_server_count (priv, local->transaction.type) == 0) {
-                        local->transaction.done (frame, this);
-                } else {
-                        int_lock->lock_cbk = local->transaction.done;
-                        afr_unlock (frame, this);
-                }
-        }
+	if (afr_lock_server_count (priv, local->transaction.type) == 0) {
+		local->transaction.done (frame, this);
+	} else {
+		int_lock->lock_cbk = local->transaction.done;
+		afr_unlock (frame, this);
+	}
 
-        return 0;
+	return 0;
 }
 
 
-void
-afr_transaction_rm_stale_children (call_frame_t *frame, xlator_t *this,
-                                   inode_t *inode, afr_transaction_type type)
+afr_inodelk_t*
+afr_get_inodelk (afr_internal_lock_t *int_lock, char *dom)
 {
-        int             i = -1;
-        int             count = 0;
-        int             read_child = -1;
-        afr_private_t   *priv = NULL;
-        afr_local_t     *local = NULL;
-        int             **pending = NULL;
-        int             idx = 0;
-        int32_t         *stale_children = NULL;
-        int32_t         *fresh_children = NULL;
-        gf_boolean_t    rm_stale_children = _gf_false;
+        afr_inodelk_t *inodelk = NULL;
+        int           i = 0;
 
-        idx = afr_index_for_transaction_type (type);
-
-        priv = this->private;
-        local = frame->local;
-        pending = local->pending;
-
-        if (local->op_ret < 0)
-                goto out;
-        fresh_children = local->fresh_children;
-        read_child = afr_inode_get_read_ctx (this, inode, fresh_children);
-        if (read_child < 0) {
-                gf_log (this->name, GF_LOG_DEBUG, "Possible split-brain "
-                        "for %s", uuid_utoa (inode->gfid));
-                goto out;
-        }
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (!afr_is_child_present (fresh_children,
-                                           priv->child_count, i))
-                        continue;
-                if (pending[i][idx])
-                        continue;
-                /* child is down or op failed on it */
-                if (!stale_children)
-                        stale_children = afr_children_create (priv->child_count);
-                if (!stale_children)
-                        goto out;
-
-                rm_stale_children = _gf_true;
-                stale_children[count++] = i;
-                gf_log (this->name, GF_LOG_DEBUG, "Removing stale child "
-                        "%d for %s", i, uuid_utoa (inode->gfid));
+        for (i = 0; int_lock->inodelk[i].domain; i++) {
+                inodelk = &int_lock->inodelk[i];
+                if (strcmp (dom, inodelk->domain) == 0)
+                        return inodelk;
         }
-
-        if (!rm_stale_children)
-                goto out;
-
-        afr_inode_rm_stale_children (this, inode, stale_children);
-out:
-        if (stale_children)
-                GF_FREE (stale_children);
-        return;
+        return NULL;
 }
 
 unsigned char*
 afr_locked_nodes_get (afr_transaction_type type, afr_internal_lock_t *int_lock)
 {
         unsigned char *locked_nodes = NULL;
+        afr_inodelk_t *inodelk = NULL;
         switch (type) {
         case AFR_DATA_TRANSACTION:
         case AFR_METADATA_TRANSACTION:
-                locked_nodes = int_lock->inode_locked_nodes;
+                inodelk = afr_get_inodelk (int_lock, int_lock->domain);
+                locked_nodes = inodelk->locked_nodes;
         break;
 
         case AFR_ENTRY_TRANSACTION:
         case AFR_ENTRY_RENAME_TRANSACTION:
-                locked_nodes = int_lock->entry_locked_nodes;
+                /*Because same set of subvols participate in all lockee
+                 * entities*/
+                locked_nodes = int_lock->lockee[0].locked_nodes;
         break;
         }
         return locked_nodes;
 }
 
+
 int
-afr_changelog_pre_op_call_count (afr_transaction_type type,
-                                 afr_internal_lock_t *int_lock,
-                                 unsigned int child_count)
+afr_changelog_call_count (afr_transaction_type type,
+			  unsigned char *pre_op_subvols,
+			  unsigned int child_count)
 {
-        int           call_count = 0;
-        unsigned char *locked_nodes = NULL;
+        int call_count = 0;
 
-        locked_nodes = afr_locked_nodes_get (type, int_lock);
-        GF_ASSERT (locked_nodes);
+	call_count = AFR_COUNT(pre_op_subvols, child_count);
 
-        call_count = afr_locked_children_count (locked_nodes, child_count);
         if (type == AFR_ENTRY_RENAME_TRANSACTION)
                 call_count *= 2;
 
         return call_count;
 }
 
-int
-afr_changelog_post_op_call_count (afr_transaction_type type,
-                                  unsigned char *pre_op,
-                                  unsigned int child_count)
+
+gf_boolean_t
+afr_txn_nothing_failed (call_frame_t *frame, xlator_t *this)
 {
-        int           call_count = 0;
+        afr_private_t *priv = NULL;
+        afr_local_t *local = NULL;
+        int i = 0;
 
-        call_count = afr_pre_op_done_children_count (pre_op, child_count);
-        if (type == AFR_ENTRY_RENAME_TRANSACTION)
-                call_count *= 2;
+        local = frame->local;
+	priv = this->private;
 
-        return call_count;
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->transaction.failed_subvols[i])
+                        return _gf_false;
+        }
+
+        return _gf_true;
 }
 
+
+void
+afr_handle_symmetric_errors (call_frame_t *frame, xlator_t *this)
+{
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	int op_errno = 0;
+	int i_errno = 0;
+	gf_boolean_t matching_errors = _gf_true;
+	int i = 0;
+
+	priv = this->private;
+	local = frame->local;
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!local->replies[i].valid)
+			continue;
+		if (local->replies[i].op_ret != -1) {
+			/* Operation succeeded on at least on subvol,
+			   so it is not a failed-everywhere situation.
+			*/
+			matching_errors = _gf_false;
+			break;
+		}
+		i_errno = local->replies[i].op_errno;
+
+		if (i_errno == ENOTCONN) {
+			/* ENOTCONN is not a symmetric error. We do not
+			   know if the operation was performed on the
+			   backend or not.
+			*/
+			matching_errors = _gf_false;
+			break;
+		}
+
+		if (!op_errno) {
+			op_errno = i_errno;
+		} else if (op_errno != i_errno) {
+			/* Mismatching op_errno's */
+			matching_errors = _gf_false;
+			break;
+		}
+	}
+
+	if (matching_errors)
+		__mark_all_success (frame, this);
+}
+
+
 int
-afr_changelog_post_op (call_frame_t *frame, xlator_t *this)
+afr_changelog_post_op_now (call_frame_t *frame, xlator_t *this)
 {
         afr_private_t * priv = this->private;
-        afr_internal_lock_t *int_lock = NULL;
-        int ret        = 0;
         int i          = 0;
-        int call_count = 0;
-
+	int ret = 0;
+	int idx = 0;
         afr_local_t *  local = NULL;
-        afr_fd_ctx_t  *fdctx = NULL;
-        dict_t        **xattr = NULL;
-        int            piggyback = 0;
-        int            index = 0;
+        dict_t        *xattr = NULL;
         int            nothing_failed = 1;
-        int32_t        changelog = 0;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        __mark_non_participant_children (local->pending, priv->child_count,
-                                         local->transaction.pre_op,
-                                         local->transaction.type);
+	gf_boolean_t   need_undirty = _gf_false;
 
-        if (local->fd)
-                afr_transaction_rm_stale_children (frame, this,
-                                                   local->fd->inode,
-                                                   local->transaction.type);
-
-        xattr = alloca (priv->child_count * sizeof (*xattr));
-        memset (xattr, 0, (priv->child_count * sizeof (*xattr)));
-        for (i = 0; i < priv->child_count; i++) {
-                xattr[i] = dict_new ();
-        }
+        local = frame->local;
+	idx = afr_index_for_transaction_type (local->transaction.type);
 
-        call_count = afr_changelog_post_op_call_count (local->transaction.type,
-                                                       local->transaction.pre_op,
-                                                       priv->child_count);
-        local->call_count = call_count;
+        nothing_failed = afr_txn_nothing_failed (frame, this);
 
-        if (local->fd)
-                fdctx = afr_fd_ctx_get (local->fd, this);
+	if (afr_changelog_pre_op_uninherit (frame, this))
+		need_undirty = _gf_false;
+	else
+		need_undirty = _gf_true;
 
-        if (call_count == 0) {
-                /* no child is up */
-                int_lock->lock_cbk = local->transaction.done;
-                afr_unlock (frame, this);
+	if (nothing_failed && !need_undirty) {
+		afr_changelog_post_op_done (frame, this);
                 goto out;
-        }
-
-        /* check if something has failed, to handle piggybacking */
-        nothing_failed = 1;
-        index = afr_index_for_transaction_type (local->transaction.type);
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->pending[i][index] == 0) {
-                        nothing_failed = 0;
-                        break;
-                }
-        }
+	}
+
+	xattr = dict_new ();
+	if (!xattr) {
+		local->op_ret = -1;
+		local->op_errno = ENOMEM;
+		afr_changelog_post_op_done (frame, this);
+		goto out;
+	}
+
+	if (need_undirty) {
+		local->dirty[idx] = hton32(-1);
+
+		ret = dict_set_static_bin (xattr, AFR_DIRTY, local->dirty,
+					   sizeof(int) * AFR_NUM_CHANGE_LOGS);
+		if (ret) {
+			local->op_ret = -1;
+			local->op_errno = ENOMEM;
+			afr_changelog_post_op_done (frame, this);
+			goto out;
+		}
+
+	}
+
+	if (!nothing_failed) {
+		for (i = 0; i < priv->child_count; i++) {
+			if (local->transaction.failed_subvols[i])
+				local->pending[i][idx] = hton32(1);
+		}
+		ret = afr_set_pending_dict (priv, xattr, local->pending);
+		if (ret < 0) {
+			local->op_ret = -1;
+			local->op_errno = ENOMEM;
+			afr_changelog_post_op_done (frame, this);
+			goto out;
+		}
+
+	}
+
+	afr_changelog_do (frame, this, xattr, afr_changelog_post_op_done);
+out:
+	if (xattr)
+                dict_unref (xattr);
 
-        index = afr_index_for_transaction_type (local->transaction.type);
-        if (local->optimistic_change_log &&
-            local->transaction.type != AFR_DATA_TRANSACTION) {
-                /* if nothing_failed, then local->pending[..] == {0 .. 0} */
-                for (i = 0; i < priv->child_count; i++) {
-                        changelog = ntoh32 (local->pending[i][index]);
-                        local->pending[i][index] = hton32 (changelog + 1);
-                }
-        }
+        return 0;
+}
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (!local->transaction.pre_op[i])
-                        continue;
-                ret = afr_set_pending_dict (priv, xattr[i], local->pending);
 
-                if (ret < 0)
-                        gf_log (this->name, GF_LOG_INFO,
-                                "failed to set pending entry");
+gf_boolean_t
+afr_changelog_pre_op_uninherit (call_frame_t *frame, xlator_t *this)
+{
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	fd_t *fd = NULL;
+	int i = 0;
+	gf_boolean_t ret = _gf_false;
+	afr_fd_ctx_t *fd_ctx = NULL;
+	int type = 0;
+
+	local = frame->local;
+	priv = this->private;
+	fd = local->fd;
+
+	type = afr_index_for_transaction_type (local->transaction.type);
+	if (type != AFR_DATA_TRANSACTION)
+		return !local->transaction.dirtied;
+
+	if (!fd)
+		return !local->transaction.dirtied;
+
+	fd_ctx = afr_fd_ctx_get (fd, this);
+	if (!fd_ctx)
+		return _gf_false;
+
+	if (local->transaction.no_uninherit)
+		return _gf_false;
+
+	/* This function must be idempotent. So check if we
+	   were called before and return the same answer again.
+
+	   It is important to keep this function idempotent for
+	   the call in afr_changelog_post_op_safe() to not have
+	   side effects on the call from afr_changelog_post_op_now()
+	*/
+	if (local->transaction.uninherit_done)
+		return local->transaction.uninherit_value;
+
+	LOCK(&fd->lock);
+	{
+		for (i = 0; i < priv->child_count; i++) {
+			if (local->transaction.pre_op[i] !=
+			    fd_ctx->pre_op_done[type][i]) {
+				ret = !local->transaction.dirtied;
+				goto unlock;
+			}
+		}
+
+		if (fd_ctx->inherited[type]) {
+			ret = _gf_true;
+			fd_ctx->inherited[type]--;
+		} else if (fd_ctx->on_disk[type]) {
+			ret = _gf_false;
+			fd_ctx->on_disk[type]--;
+		} else {
+			/* ASSERT */
+			ret = _gf_false;
+		}
+
+		if (!fd_ctx->inherited[type] && !fd_ctx->on_disk[type]) {
+			for (i = 0; i < priv->child_count; i++)
+				fd_ctx->pre_op_done[type][i] = 0;
+		}
+	}
+unlock:
+	UNLOCK(&fd->lock);
+
+	local->transaction.uninherit_done = _gf_true;
+	local->transaction.uninherit_value = ret;
+
+	return ret;
+}
 
 
-                switch (local->transaction.type) {
-                case AFR_DATA_TRANSACTION:
-                {
-                        if (!fdctx) {
-                                STACK_WIND (frame, afr_changelog_post_op_cbk,
-                                            priv->children[i],
-                                            priv->children[i]->fops->xattrop,
-                                            &local->loc,
-                                            GF_XATTROP_ADD_ARRAY, xattr[i],
-                                            NULL);
-                                break;
-                        }
+gf_boolean_t
+afr_changelog_pre_op_inherit (call_frame_t *frame, xlator_t *this)
+{
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	fd_t *fd = NULL;
+	int i = 0;
+	gf_boolean_t ret = _gf_false;
+	afr_fd_ctx_t *fd_ctx = NULL;
+	int type = 0;
+
+	local = frame->local;
+	priv = this->private;
+	fd = local->fd;
+
+	if (local->transaction.type != AFR_DATA_TRANSACTION)
+		return _gf_false;
+
+	type = afr_index_for_transaction_type (local->transaction.type);
+
+	if (!fd)
+		return _gf_false;
+
+	fd_ctx = afr_fd_ctx_get (fd, this);
+	if (!fd_ctx)
+		return _gf_false;
+
+	LOCK(&fd->lock);
+	{
+		if (!fd_ctx->on_disk[type]) {
+			/* nothing to inherit yet */
+			ret = _gf_false;
+			goto unlock;
+		}
+
+		for (i = 0; i < priv->child_count; i++) {
+			if (local->transaction.pre_op[i] !=
+			    fd_ctx->pre_op_done[type][i]) {
+				/* either inherit exactly, or don't */
+				ret = _gf_false;
+				goto unlock;
+			}
+		}
+
+		fd_ctx->inherited[type]++;
+
+		ret = _gf_true;
+
+		local->transaction.inherited = _gf_true;
+	}
+unlock:
+	UNLOCK(&fd->lock);
+
+	return ret;
+}
 
-                        LOCK (&local->fd->lock);
-                        {
-                                piggyback = 0;
-                                if (fdctx->pre_op_piggyback[i]) {
-                                        fdctx->pre_op_piggyback[i]--;
-                                        piggyback = 1;
-                                }
-                        }
-                        UNLOCK (&local->fd->lock);
 
-                        if (piggyback && !nothing_failed)
-                                ret = afr_set_piggyback_dict (priv, xattr[i],
-                                                              local->pending,
-                                                              local->transaction.type);
+gf_boolean_t
+afr_changelog_pre_op_update (call_frame_t *frame, xlator_t *this)
+{
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	fd_t *fd = NULL;
+	afr_fd_ctx_t *fd_ctx = NULL;
+	int i = 0;
+	gf_boolean_t ret = _gf_false;
+	int type = 0;
+
+	local = frame->local;
+	priv = this->private;
+	fd = local->fd;
+
+	if (!fd)
+		return _gf_false;
+
+	fd_ctx = afr_fd_ctx_get (fd, this);
+	if (!fd_ctx)
+		return _gf_false;
+
+	if (local->transaction.inherited)
+		/* was already inherited in afr_changelog_pre_op */
+		return _gf_false;
+
+	if (!local->transaction.dirtied)
+		return _gf_false;
+
+        if (!afr_txn_nothing_failed (frame, this))
+		return _gf_false;
+
+	type = afr_index_for_transaction_type (local->transaction.type);
+
+	ret = _gf_false;
+
+	LOCK(&fd->lock);
+	{
+		if (!fd_ctx->on_disk[type]) {
+			for (i = 0; i < priv->child_count; i++)
+				fd_ctx->pre_op_done[type][i] =
+					local->transaction.pre_op[i];
+		} else {
+			for (i = 0; i < priv->child_count; i++)
+				if (fd_ctx->pre_op_done[type][i] !=
+				    local->transaction.pre_op[i]) {
+					local->transaction.no_uninherit = 1;
+					goto unlock;
+				}
+		}
+		fd_ctx->on_disk[type]++;
+
+		ret = _gf_true;
+	}
+unlock:
+	UNLOCK(&fd->lock);
+
+	return ret;
+}
 
-                        if (nothing_failed && piggyback) {
-                                afr_changelog_post_op_cbk (frame, (void *)(long)i,
-                                                           this, 1, 0, xattr[i], NULL);
-                        } else {
-                                __mark_pre_op_undone_on_fd (frame, this, i);
-                                STACK_WIND_COOKIE (frame,
-                                                   afr_changelog_post_op_cbk,
-                                                   (void *) (long) i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->fxattrop,
-                                                   local->fd,
-                                                   GF_XATTROP_ADD_ARRAY, xattr[i],
-                                                   NULL);
-                        }
-                }
-                break;
-                case AFR_METADATA_TRANSACTION:
-                {
-                        if (nothing_failed) {
-                                afr_changelog_post_op_cbk (frame, (void *)(long)i,
-                                                           this, 1, 0, xattr[i],
-                                                           NULL);
-                                break;
-                        }
 
-                        if (local->fd)
-                                STACK_WIND (frame, afr_changelog_post_op_cbk,
-                                            priv->children[i],
-                                            priv->children[i]->fops->fxattrop,
-                                            local->fd,
-                                            GF_XATTROP_ADD_ARRAY, xattr[i],
-                                            NULL);
-                        else
-                                STACK_WIND (frame, afr_changelog_post_op_cbk,
-                                            priv->children[i],
-                                            priv->children[i]->fops->xattrop,
-                                            &local->loc,
-                                            GF_XATTROP_ADD_ARRAY, xattr[i],
-                                            NULL);
-                }
-                break;
+int
+afr_changelog_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		   int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
+{
+        afr_local_t *local = NULL;
+        int call_count = -1;
 
-                case AFR_ENTRY_RENAME_TRANSACTION:
-                {
-                        if (nothing_failed) {
-                                afr_changelog_post_op_cbk (frame, (void *)(long)i,
-                                                           this, 1, 0, xattr[i],
-                                                           NULL);
-                        } else {
-                                STACK_WIND_COOKIE (frame, afr_changelog_post_op_cbk,
-                                                   (void *) (long) i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->xattrop,
-                                                   &local->transaction.new_parent_loc,
-                                                   GF_XATTROP_ADD_ARRAY, xattr[i],
-                                                   NULL);
-                        }
-                        call_count--;
-                }
+        local = frame->local;
 
-                /*
-                  set it again because previous stack_wind
-                  might have already returned (think of case
-                  where subvolume is posix) and would have
-                  used the dict as placeholder for return
-                  value
-                */
+	if (op_ret == -1)
+		afr_transaction_fop_failed (frame, this, (long) cookie);
 
-                ret = afr_set_pending_dict (priv, xattr[i], local->pending);
+	call_count = afr_frame_return (frame);
 
-                if (ret < 0)
-                        gf_log (this->name, GF_LOG_INFO,
-                                "failed to set pending entry");
+        if (call_count == 0)
+		local->transaction.changelog_resume (frame, this);
 
-                /* fall through */
+        return 0;
+}
 
-                case AFR_ENTRY_TRANSACTION:
-                {
-                        if (nothing_failed) {
-                                afr_changelog_post_op_cbk (frame, (void *)(long)i,
-                                                           this, 1, 0, xattr[i],
-                                                           NULL);
-                                break;
-                        }
 
-                        if (local->fd)
-                                STACK_WIND (frame, afr_changelog_post_op_cbk,
-                                            priv->children[i],
-                                            priv->children[i]->fops->fxattrop,
-                                            local->fd,
-                                            GF_XATTROP_ADD_ARRAY, xattr[i],
-                                            NULL);
-                        else
-                                STACK_WIND (frame, afr_changelog_post_op_cbk,
-                                            priv->children[i],
-                                            priv->children[i]->fops->xattrop,
-                                            &local->transaction.parent_loc,
-                                            GF_XATTROP_ADD_ARRAY, xattr[i],
-                                            NULL);
-                }
-                break;
-                }
+int
+afr_changelog_do (call_frame_t *frame, xlator_t *this, dict_t *xattr,
+		  afr_changelog_resume_t changelog_resume)
+{
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	int i = 0;
+	int call_count = 0;
 
-                if (!--call_count)
-                        break;
-        }
+	local = frame->local;
+	priv = this->private;
 
-out:
-        for (i = 0; i < priv->child_count; i++) {
-                dict_unref (xattr[i]);
-        }
+        call_count = afr_changelog_call_count (local->transaction.type,
+					       local->transaction.pre_op,
+					       priv->child_count);
 
-        return 0;
-}
+	if (call_count == 0) {
+		changelog_resume (frame, this);
+		return 0;
+	}
 
+	local->call_count = call_count;
 
-int32_t
-afr_changelog_pre_op_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, dict_t *xattr,
-                          dict_t *xdata)
-{
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = this->private;
-        int call_count  = -1;
-        int child_index = (long) cookie;
+	local->transaction.changelog_resume = changelog_resume;
 
-        local = frame->local;
+        for (i = 0; i < priv->child_count; i++) {
+                if (!local->transaction.pre_op[i])
+                        continue;
 
-        LOCK (&frame->lock);
-        {
-                switch (op_ret) {
-                case 0:
-                        __mark_pre_op_done_on_fd (frame, this, child_index);
-                        //fallthrough we need to mark the pre_op
-                case 1:
-                        local->transaction.pre_op[child_index] = 1;
-                        /* special op_ret for piggyback */
-                        break;
-                case -1:
-                        if (op_errno == ENOTSUP) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "xattrop not supported by %s",
-                                        priv->children[child_index]->name);
-                                local->op_ret = -1;
-
-                        } else if (!child_went_down (op_ret, op_errno)) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "xattrop failed on child %s: %s",
-                                        priv->children[child_index]->name,
-                                        strerror (op_errno));
+                switch (local->transaction.type) {
+                case AFR_DATA_TRANSACTION:
+                case AFR_METADATA_TRANSACTION:
+                        if (!local->fd) {
+                                STACK_WIND_COOKIE (frame, afr_changelog_cbk,
+						   (void *) (long) i,
+						   priv->children[i],
+						   priv->children[i]->fops->xattrop,
+						   &local->loc,
+						   GF_XATTROP_ADD_ARRAY, xattr,
+						   NULL);
+                        } else {
+                                STACK_WIND_COOKIE (frame, afr_changelog_cbk,
+						   (void *) (long) i,
+						   priv->children[i],
+						   priv->children[i]->fops->fxattrop,
+						   local->fd,
+						   GF_XATTROP_ADD_ARRAY, xattr,
+						   NULL);
                         }
-                        local->op_errno = op_errno;
-                        break;
-                }
+			break;
+                case AFR_ENTRY_RENAME_TRANSACTION:
 
-                call_count = --local->call_count;
-        }
-        UNLOCK (&frame->lock);
+			STACK_WIND_COOKIE (frame, afr_changelog_cbk,
+					   (void *) (long) i,
+					   priv->children[i],
+					   priv->children[i]->fops->xattrop,
+					   &local->transaction.new_parent_loc,
+					   GF_XATTROP_ADD_ARRAY, xattr,
+					   NULL);
+                        call_count--;
 
-        if (call_count == 0) {
-                if ((local->op_ret == -1) &&
-                    (local->op_errno == ENOTSUP)) {
-                        local->transaction.resume (frame, this);
-                } else {
-                        __mark_all_success (local->pending, priv->child_count,
-                                            local->transaction.type);
+                /* fall through */
 
-                        afr_pid_restore (frame);
+                case AFR_ENTRY_TRANSACTION:
+                        if (local->fd)
+                                STACK_WIND_COOKIE (frame, afr_changelog_cbk,
+						   (void *) (long) i,
+						   priv->children[i],
+						   priv->children[i]->fops->fxattrop,
+						   local->fd,
+						   GF_XATTROP_ADD_ARRAY, xattr,
+						   NULL);
+                        else
+                                STACK_WIND_COOKIE (frame, afr_changelog_cbk,
+						   (void *) (long) i,
+						   priv->children[i],
+						   priv->children[i]->fops->xattrop,
+						   &local->transaction.parent_loc,
+						   GF_XATTROP_ADD_ARRAY, xattr,
+						   NULL);
+			break;
+		}
 
-                        local->transaction.fop (frame, this);
-                }
+                if (!--call_count)
+                        break;
         }
 
-        return 0;
+	return 0;
 }
 
+
 int
 afr_changelog_pre_op (call_frame_t *frame, xlator_t *this)
 {
@@ -782,202 +816,122 @@ afr_changelog_pre_op (call_frame_t *frame, xlator_t *this)
         int i = 0;
         int ret = 0;
         int call_count = 0;
-        dict_t **xattr = NULL;
-        afr_fd_ctx_t *fdctx = NULL;
+	int op_errno = 0;
         afr_local_t *local = NULL;
-        int          piggyback = 0;
         afr_internal_lock_t *int_lock = NULL;
         unsigned char       *locked_nodes = NULL;
+	unsigned char       *pending_subvols = NULL;
+	int idx = -1;
+	gf_boolean_t pre_nop = _gf_true;
+	dict_t *xdata_req = NULL;
 
         local = frame->local;
         int_lock = &local->internal_lock;
-
-        xattr = alloca (priv->child_count * sizeof (*xattr));
-        memset (xattr, 0, (priv->child_count * sizeof (*xattr)));
-
-        for (i = 0; i < priv->child_count; i++) {
-                xattr[i] = dict_new ();
-        }
-
-        call_count = afr_changelog_pre_op_call_count (local->transaction.type,
-                                                      int_lock,
-                                                      priv->child_count);
-        if (call_count == 0) {
-                local->internal_lock.lock_cbk =
-                        local->transaction.done;
-                afr_unlock (frame, this);
-                goto out;
-        }
-
-        local->call_count = call_count;
-
-        __mark_all_pending (local->pending, priv->child_count,
-                            local->transaction.type);
-
-        if (local->fd)
-                fdctx = afr_fd_ctx_get (local->fd, this);
+	idx = afr_index_for_transaction_type (local->transaction.type);
 
         locked_nodes = afr_locked_nodes_get (local->transaction.type, int_lock);
-        for (i = 0; i < priv->child_count; i++) {
-                if (!locked_nodes[i])
-                        continue;
-                ret = afr_set_pending_dict (priv, xattr[i], local->pending);
 
-                if (ret < 0)
-                        gf_log (this->name, GF_LOG_INFO,
-                                "failed to set pending entry");
+	pending_subvols = alloca0 (priv->child_count);
 
+	for (i = 0; i < priv->child_count; i++) {
+		if (locked_nodes[i]) {
+			local->transaction.pre_op[i] = 1;
+			call_count++;
+		} else {
+			pending_subvols[i] = 1;
+		}
+	}
 
-                switch (local->transaction.type) {
-                case AFR_DATA_TRANSACTION:
-                {
-                        if (!fdctx) {
-                                STACK_WIND_COOKIE (frame,
-                                                   afr_changelog_pre_op_cbk,
-                                                   (void *) (long) i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->xattrop,
-                                                   &(local->loc),
-                                                   GF_XATTROP_ADD_ARRAY, xattr[i],
-                                                   NULL);
-                                break;
-                        }
-
-                        LOCK (&local->fd->lock);
-                        {
-                                piggyback = 0;
-                                if (fdctx->pre_op_done[i]) {
-                                        fdctx->pre_op_piggyback[i]++;
-                                        piggyback = 1;
-                                        fdctx->hit++;
-                                } else {
-                                        fdctx->miss++;
-                                }
-                        }
-                        UNLOCK (&local->fd->lock);
-
-                        if (piggyback)
-                                afr_changelog_pre_op_cbk (frame, (void *)(long)i,
-                                                          this, 1, 0, xattr[i],
-                                                          NULL);
-                        else
-                                STACK_WIND_COOKIE (frame,
-                                                   afr_changelog_pre_op_cbk,
-                                                   (void *) (long) i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->fxattrop,
-                                                   local->fd,
-                                                   GF_XATTROP_ADD_ARRAY, xattr[i],
-                                                   NULL);
-                }
-                break;
-                case AFR_METADATA_TRANSACTION:
-                {
-                        if (local->optimistic_change_log) {
-                                afr_changelog_pre_op_cbk (frame, (void *)(long)i,
-                                                          this, 1, 0, xattr[i],
-                                                          NULL);
-                                break;
-                        }
-
-                        if (local->fd)
-                                STACK_WIND_COOKIE (frame,
-                                                   afr_changelog_pre_op_cbk,
-                                                   (void *) (long) i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->fxattrop,
-                                                   local->fd,
-                                                   GF_XATTROP_ADD_ARRAY, xattr[i],
-                                                   NULL);
-                        else
-                                STACK_WIND_COOKIE (frame,
-                                                   afr_changelog_pre_op_cbk,
-                                                   (void *) (long) i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->xattrop,
-                                                   &(local->loc),
-                                                   GF_XATTROP_ADD_ARRAY, xattr[i],
-                                                   NULL);
-                }
-                break;
+	/* TBD: quorum check w/ call_count */
 
-                case AFR_ENTRY_RENAME_TRANSACTION:
-                {
-                        if (local->optimistic_change_log) {
-                                afr_changelog_pre_op_cbk (frame, (void *)(long)i,
-                                                          this, 1, 0, xattr[i],
-                                                          NULL);
-                        } else {
-                                STACK_WIND_COOKIE (frame,
-                                                   afr_changelog_pre_op_cbk,
-                                                   (void *) (long) i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->xattrop,
-                                                   &local->transaction.new_parent_loc,
-                                                   GF_XATTROP_ADD_ARRAY, xattr[i],
-                                                   NULL);
-                        }
-
-                        call_count--;
-                }
-
-
-                /*
-                  set it again because previous stack_wind
-                  might have already returned (think of case
-                  where subvolume is posix) and would have
-                  used the dict as placeholder for return
-                  value
-                */
-
-                ret = afr_set_pending_dict (priv, xattr[i], local->pending);
-
-                if (ret < 0)
-                        gf_log (this->name, GF_LOG_INFO,
-                                "failed to set pending entry");
-
-                /* fall through */
+        if (call_count == 0) {
+		op_errno = ENOTCONN;
+		goto err;
+	}
+
+	xdata_req = dict_new();
+	if (!xdata_req) {
+		op_errno = ENOMEM;
+		goto err;
+	}
+
+	pre_nop = _gf_true;
+
+	if (afr_changelog_pre_op_inherit (frame, this))
+		goto next;
+
+	if (call_count < priv->child_count) {
+		/* For subvols we are not performing operation on,
+		   mark them as pending up-front along with the FOP
+		   so that we can safely defer unmarking dirty until
+		   later.
+		*/
+		for (i = 0; i < priv->child_count; i++) {
+			if (pending_subvols[i])
+				local->pending[i][idx] = hton32(1);
+		}
+		ret = afr_set_pending_dict (priv, xdata_req,
+					    local->pending);
+		if (ret < 0) {
+			op_errno = ENOMEM;
+			goto err;
+		}
+		pre_nop = _gf_false;
+	}
+
+	if (call_count > 1 &&
+	    (local->transaction.type == AFR_DATA_TRANSACTION ||
+	     !local->optimistic_change_log)) {
+
+		/* If we are performing change on only one subvol, no
+		   need to mark dirty, because we are setting the pending
+		   counts already anyways
+		*/
+		local->dirty[idx] = hton32(1);
+
+		ret = dict_set_static_bin (xdata_req, AFR_DIRTY, local->dirty,
+					   sizeof(int) * AFR_NUM_CHANGE_LOGS);
+		if (ret) {
+			op_errno = ENOMEM;
+			goto err;
+		}
+
+		pre_nop = _gf_false;
+		local->transaction.dirtied = 1;
+	}
+
+	if (pre_nop)
+		goto next;
+
+	if (!local->pre_op_compat) {
+		dict_copy (xdata_req, local->xdata_req);
+		goto next;
+	}
+
+	afr_changelog_do (frame, this, xdata_req, afr_transaction_perform_fop);
+
+	if (xdata_req)
+		dict_unref (xdata_req);
+
+	return 0;
+next:
+	afr_transaction_perform_fop (frame, this);
+
+	if (xdata_req)
+		dict_unref (xdata_req);
 
-                case AFR_ENTRY_TRANSACTION:
-                {
-                        if (local->optimistic_change_log) {
-                                afr_changelog_pre_op_cbk (frame, (void *)(long)i,
-                                                          this, 1, 0, xattr[i],
-                                                          NULL);
-                                break;
-                        }
+        return 0;
+err:
+	local->internal_lock.lock_cbk = local->transaction.done;
+	local->op_ret = -1;
+	local->op_errno = op_errno;
 
-                        if (local->fd)
-                                STACK_WIND_COOKIE (frame,
-                                                   afr_changelog_pre_op_cbk,
-                                                   (void *) (long) i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->fxattrop,
-                                                   local->fd,
-                                                   GF_XATTROP_ADD_ARRAY, xattr[i],
-                                                   NULL);
-                        else
-                                STACK_WIND_COOKIE (frame,
-                                                   afr_changelog_pre_op_cbk,
-                                                   (void *) (long) i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->xattrop,
-                                                   &local->transaction.parent_loc,
-                                                   GF_XATTROP_ADD_ARRAY, xattr[i],
-                                                   NULL);
-                }
-                break;
-                }
+	afr_unlock (frame, this);
 
-                if (!--call_count)
-                        break;
-        }
-out:
-        for (i = 0; i < priv->child_count; i++) {
-                dict_unref (xattr[i]);
-        }
+	if (xdata_req)
+		dict_unref (xdata_req);
 
-        return 0;
+	return 0;
 }
 
 
@@ -1019,7 +973,6 @@ afr_post_nonblocking_inodelk_cbk (call_frame_t *frame, xlator_t *this)
                 gf_log (this->name, GF_LOG_DEBUG,
                         "Non blocking inodelks failed. Proceeding to blocking");
                 int_lock->lock_cbk = afr_post_blocking_inodelk_cbk;
-                afr_set_lk_owner (frame, this, frame->root);
                 afr_blocking_lock (frame, this);
         } else {
 
@@ -1127,12 +1080,14 @@ int
 afr_set_transaction_flock (afr_local_t *local)
 {
         afr_internal_lock_t *int_lock = NULL;
+        afr_inodelk_t       *inodelk  = NULL;
 
         int_lock = &local->internal_lock;
+        inodelk = afr_get_inodelk (int_lock, int_lock->domain);
 
-        int_lock->lk_flock.l_len   = local->transaction.len;
-        int_lock->lk_flock.l_start = local->transaction.start;
-        int_lock->lk_flock.l_type  = F_WRLCK;
+        inodelk->flock.l_len   = local->transaction.len;
+        inodelk->flock.l_start = local->transaction.start;
+        inodelk->flock.l_type  = F_WRLCK;
 
         return 0;
 }
@@ -1147,6 +1102,7 @@ afr_lock_rec (call_frame_t *frame, xlator_t *this)
         int_lock = &local->internal_lock;
 
         int_lock->transaction_lk_type = AFR_TRANSACTION_LK;
+        int_lock->domain = this->name;
 
         switch (local->transaction.type) {
         case AFR_DATA_TRANSACTION:
@@ -1160,8 +1116,8 @@ afr_lock_rec (call_frame_t *frame, xlator_t *this)
 
         case AFR_ENTRY_RENAME_TRANSACTION:
 
-                int_lock->lock_cbk = afr_post_blocking_rename_cbk;
-                afr_blocking_lock (frame, this);
+                int_lock->lock_cbk = afr_post_nonblocking_entrylk_cbk;
+                afr_nonblocking_entrylk (frame, this);
                 break;
 
         case AFR_ENTRY_TRANSACTION:
@@ -1183,12 +1139,6 @@ afr_lock_rec (call_frame_t *frame, xlator_t *this)
 int
 afr_lock (call_frame_t *frame, xlator_t *this)
 {
-        afr_pid_save (frame);
-
-        frame->root->pid = (long) frame->root;
-
-        afr_set_lk_owner (frame, this, frame->root);
-
         afr_set_lock_number (frame, this);
 
         return afr_lock_rec (frame, this);
@@ -1200,52 +1150,427 @@ afr_lock (call_frame_t *frame, xlator_t *this)
 int
 afr_internal_lock_finish (call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t   *local = NULL;
-        afr_private_t *priv  = NULL;
+        if (__fop_changelog_needed (frame, this)) {
+                afr_changelog_pre_op (frame, this);
+        } else {
+                afr_transaction_perform_fop (frame, this);
+        }
+
+        return 0;
+}
+
+
+void
+afr_set_delayed_post_op (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t    *local = NULL;
+        afr_private_t  *priv = NULL;
+
+        /* call this function from any of the related optimizations
+           which benefit from delaying post op are enabled, namely:
+
+           - changelog piggybacking
+           - eager locking
+        */
+
+        priv = this->private;
+        if (!priv)
+                return;
+
+        if (!priv->post_op_delay_secs)
+                return;
 
-        priv  = this->private;
         local = frame->local;
+        if (!local->transaction.eager_lock_on)
+                return;
 
-        /*  Perform fops with the lk-owner from top xlator.
-         *  Eg: lk-owner of posix-lk and flush should be same,
-         *  flush cant clear the  posix-lks without that lk-owner.
+        if (!local)
+                return;
+
+        if (!local->fd)
+                return;
+
+        if (local->op == GF_FOP_WRITE)
+                local->delayed_post_op = _gf_true;
+}
+
+gf_boolean_t
+afr_are_multiple_fds_opened (fd_t *fd, xlator_t *this)
+{
+        afr_fd_ctx_t *fd_ctx = NULL;
+
+        if (!fd) {
+                /* If false is returned, it may keep on taking eager-lock
+                 * which may lead to starvation, so return true to avoid that.
+                 */
+                gf_log_callingfn (this->name, GF_LOG_ERROR, "Invalid fd");
+                return _gf_true;
+        }
+        /* Lets say mount1 has eager-lock(full-lock) and after the eager-lock
+         * is taken mount2 opened the same file, it won't be able to
+         * perform any data operations until mount1 releases eager-lock.
+         * To avoid such scenario do not enable eager-lock for this transaction
+         * if open-fd-count is > 1
          */
-        frame->root->lk_owner = local->transaction.main_frame->root->lk_owner;
-        if (__fop_changelog_needed (frame, this)) {
-                afr_changelog_pre_op (frame, this);
-        } else {
-                __mark_all_success (local->pending, priv->child_count,
-                                    local->transaction.type);
 
-                afr_pid_restore (frame);
+        fd_ctx = afr_fd_ctx_get (fd, this);
+        if (!fd_ctx)
+                return _gf_true;
+
+        if (fd_ctx->open_fd_count > 1)
+                return _gf_true;
+
+        return _gf_false;
+}
+
+
+gf_boolean_t
+is_afr_delayed_changelog_post_op_needed (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t      *local = NULL;
+        gf_boolean_t      res = _gf_false;
+
+        local = frame->local;
+        if (!local)
+                goto out;
+
+        if (!local->delayed_post_op)
+                goto out;
+
+        //Mark pending changelog ASAP
+        if (!afr_txn_nothing_failed (frame, this))
+                goto out;
+
+        if (local->fd && afr_are_multiple_fds_opened (local->fd, this))
+                goto out;
+
+        res = _gf_true;
+out:
+        return res;
+}
+
+
+void
+afr_delayed_changelog_post_op (xlator_t *this, call_frame_t *frame, fd_t *fd,
+                               call_stub_t *stub);
+
+void
+afr_delayed_changelog_wake_up_cbk (void *data)
+{
+        fd_t           *fd = NULL;
+
+        fd = data;
+
+        afr_delayed_changelog_wake_up (THIS, fd);
+}
+
+
+/* SET operation */
+int
+afr_fd_report_unstable_write (xlator_t *this, fd_t *fd)
+{
+        afr_fd_ctx_t *fdctx = NULL;
+
+        fdctx = afr_fd_ctx_get (fd, this);
+
+        LOCK(&fd->lock);
+        {
+                fdctx->witnessed_unstable_write = _gf_true;
+        }
+        UNLOCK(&fd->lock);
+
+        return 0;
+}
+
+/* TEST and CLEAR operation */
+gf_boolean_t
+afr_fd_has_witnessed_unstable_write (xlator_t *this, fd_t *fd)
+{
+        afr_fd_ctx_t *fdctx = NULL;
+        gf_boolean_t witness = _gf_false;
+
+        fdctx = afr_fd_ctx_get (fd, this);
+        if (!fdctx)
+                return _gf_true;
+
+        LOCK(&fd->lock);
+        {
+                if (fdctx->witnessed_unstable_write) {
+                        witness = _gf_true;
+                        fdctx->witnessed_unstable_write = _gf_false;
+                }
+        }
+        UNLOCK (&fd->lock);
+
+        return witness;
+}
+
+
+int
+afr_changelog_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                         int op_ret, int op_errno, struct iatt *pre,
+                         struct iatt *post, dict_t *xdata)
+{
+        afr_private_t *priv = NULL;
+        int child_index = (long) cookie;
+        int call_count = -1;
+        afr_local_t *local = NULL;
+
+        priv = this->private;
+        local = frame->local;
+
+        if (op_ret != 0) {
+                /* Failure of fsync() is as good as failure of previous
+                   write(). So treat it like one.
+		*/
+                gf_log (this->name, GF_LOG_WARNING,
+                        "fsync(%s) failed on subvolume %s. Transaction was %s",
+                        uuid_utoa (local->fd->inode->gfid),
+                        priv->children[child_index]->name,
+                        gf_fop_list[local->op]);
+
+                afr_transaction_fop_failed (frame, this, child_index);
+	}
+
+	call_count = afr_frame_return (frame);
+
+	if (call_count == 0)
+		afr_changelog_post_op_now (frame, this);
+
+	return 0;
+}
+
+
+int
+afr_changelog_fsync (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t *local = NULL;
+        int i = 0;
+        int call_count = 0;
+        afr_private_t *priv = NULL;
+        dict_t *xdata = NULL;
+        GF_UNUSED int ret = -1;
+
+        local = frame->local;
+        priv = this->private;
+
+        call_count = AFR_COUNT (local->transaction.pre_op, priv->child_count);
+
+        if (!call_count) {
+                /* will go straight to unlock */
+                afr_changelog_post_op_now (frame, this);
+                return 0;
+        }
+
+        local->call_count = call_count;
+
+        xdata = dict_new();
+        if (xdata)
+                ret = dict_set_int32 (xdata, "batch-fsync", 1);
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (!local->transaction.pre_op[i])
+                        continue;
+
+                STACK_WIND_COOKIE (frame, afr_changelog_fsync_cbk,
+				   (void *) (long) i, priv->children[i],
+				   priv->children[i]->fops->fsync, local->fd,
+				   1, xdata);
+                if (!--call_count)
+                        break;
+        }
+
+        if (xdata)
+                dict_unref (xdata);
+
+        return 0;
+}
+
+
+int
+afr_changelog_post_op_safe (call_frame_t *frame, xlator_t *this)
+{
+	afr_local_t    *local = NULL;
+        afr_private_t  *priv = NULL;
+
+	local = frame->local;
+        priv = this->private;
+
+        if (!local->fd || local->transaction.type != AFR_DATA_TRANSACTION) {
+                afr_changelog_post_op_now (frame, this);
+                return 0;
+        }
+
+        if (afr_changelog_pre_op_uninherit (frame, this) &&
+	    afr_txn_nothing_failed (frame, this)) {
+                /* just detected that this post-op is about to
+                   be optimized away as a new write() has
+                   already piggybacked on this frame's changelog.
+                   */
+                afr_changelog_post_op_now (frame, this);
+                return 0;
+        }
+
+        /* Calling afr_changelog_post_op_now() now will result in
+           issuing ->[f]xattrop().
+
+           Performing a hard POST-OP (->[f]xattrop() FOP) is a more
+           responsible operation that what it might appear on the surface.
+
+           The changelog of a file (in the xattr of the file on the server)
+           stores information (pending count) about the state of the file
+           on the OTHER server. This changelog is blindly trusted, and must
+           therefore be updated in such a way it remains trustworthy. This
+           implies that decrementing the pending count (essentially "clearing
+           the dirty flag") must be done STRICTLY after we are sure that the
+           operation on the other server has reached stable storage.
+
+           While the backend filesystem on that server will eventually flush
+           it to stable storage, we (being in userspace) have no mechanism
+           to get notified when the write became "stable".
+
+           This means we need take matter into our own hands and issue an
+           fsync() EVEN IF THE APPLICATION WAS PERFORMING UNSTABLE WRITES,
+           and get an acknowledgement for it. And we need to wait for the
+           fsync() acknowledgement before initiating the hard POST-OP.
+
+           However if the FD itself was opened in O_SYNC or O_DSYNC then
+           we are already guaranteed that the writes were made stable as
+           part of the FOP itself. The same holds true for NFS stable
+           writes which happen on an anonymous FD with O_DSYNC or O_SYNC
+           flag set in the writev() @flags param. For all other write types,
+           mark a flag in the fdctx whenever an unstable write is witnessed.
+           */
+
+        if (!afr_fd_has_witnessed_unstable_write (this, local->fd)) {
+                afr_changelog_post_op_now (frame, this);
+                return 0;
+        }
 
-                local->transaction.fop (frame, this);
+        /* Check whether users want durability and perform fsync/post-op
+         * accordingly.
+         */
+        if (priv->ensure_durability) {
+                /* Time to fsync() */
+                afr_changelog_fsync (frame, this);
+        } else {
+                afr_changelog_post_op_now (frame, this);
         }
 
         return 0;
 }
 
 
+void
+afr_delayed_changelog_post_op (xlator_t *this, call_frame_t *frame, fd_t *fd,
+                               call_stub_t *stub)
+{
+	afr_fd_ctx_t      *fd_ctx = NULL;
+	call_frame_t      *prev_frame = NULL;
+	struct timespec    delta = {0, };
+	afr_private_t     *priv = NULL;
+	afr_local_t       *local = NULL;
+
+	priv = this->private;
+
+	fd_ctx = afr_fd_ctx_get (fd, this);
+	if (!fd_ctx)
+                goto out;
+
+	delta.tv_sec = priv->post_op_delay_secs;
+	delta.tv_nsec = 0;
+
+	pthread_mutex_lock (&fd_ctx->delay_lock);
+	{
+		prev_frame = fd_ctx->delay_frame;
+		fd_ctx->delay_frame = NULL;
+		if (fd_ctx->delay_timer)
+			gf_timer_call_cancel (this->ctx, fd_ctx->delay_timer);
+		fd_ctx->delay_timer = NULL;
+		if (!frame)
+			goto unlock;
+		fd_ctx->delay_timer = gf_timer_call_after (this->ctx, delta,
+							   afr_delayed_changelog_wake_up_cbk,
+							   fd);
+		fd_ctx->delay_frame = frame;
+	}
+unlock:
+	pthread_mutex_unlock (&fd_ctx->delay_lock);
+
+out:
+	if (prev_frame) {
+		local = prev_frame->local;
+		local->transaction.resume_stub = stub;
+		afr_changelog_post_op_now (prev_frame, this);
+	} else if (stub) {
+		call_resume (stub);
+	}
+}
+
+
+void
+afr_changelog_post_op (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t  *local = NULL;
+
+        local = frame->local;
+
+        if (is_afr_delayed_changelog_post_op_needed (frame, this))
+                afr_delayed_changelog_post_op (this, frame, local->fd, NULL);
+        else
+                afr_changelog_post_op_safe (frame, this);
+}
+
+
+
+/* Wake up the sleeping/delayed post-op, and also register
+   a stub to have it resumed after this transaction
+   completely finishes.
+
+   The @stub gets saved in @local and gets resumed in
+   afr_local_cleanup()
+   */
+void
+afr_delayed_changelog_wake_resume (xlator_t *this, fd_t *fd, call_stub_t *stub)
+{
+        afr_delayed_changelog_post_op (this, NULL, fd, stub);
+}
+
+
+void
+afr_delayed_changelog_wake_up (xlator_t *this, fd_t *fd)
+{
+        afr_delayed_changelog_post_op (this, NULL, fd, NULL);
+}
+
+
 int
 afr_transaction_resume (call_frame_t *frame, xlator_t *this)
 {
-        afr_internal_lock_t *int_lock = NULL;
         afr_local_t         *local    = NULL;
-        afr_private_t       *priv     = NULL;
 
         local    = frame->local;
-        int_lock = &local->internal_lock;
-        priv     = this->private;
+
+        if (local->transaction.eager_lock_on) {
+                /* We don't need to retain "local" in the
+                   fd list anymore, writes to all subvols
+                   are finished by now */
+                afr_remove_eager_lock_stub (local);
+        }
+
+        afr_restore_lk_owner (frame);
+
+        afr_handle_symmetric_errors (frame, this);
+
+	if (!local->pre_op_compat)
+		/* new mode, pre-op was done along
+		   with OP */
+		afr_changelog_pre_op_update (frame, this);
 
         if (__fop_changelog_needed (frame, this)) {
                 afr_changelog_post_op (frame, this);
         } else {
-                if (afr_lock_server_count (priv, local->transaction.type) == 0) {
-                        local->transaction.done (frame, this);
-                } else {
-                        int_lock->lock_cbk = local->transaction.done;
-                        afr_unlock (frame, this);
-                }
+		afr_changelog_post_op_done (frame, this);
         }
 
         return 0;
@@ -1257,16 +1582,96 @@ afr_transaction_resume (call_frame_t *frame, xlator_t *this)
  */
 
 void
-afr_transaction_fop_failed (call_frame_t *frame, xlator_t *this, int child_index)
+afr_transaction_fop_failed (call_frame_t *frame, xlator_t *this,
+                            int child_index)
 {
         afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
 
         local = frame->local;
-        priv  = this->private;
 
-        __mark_child_dead (local->pending, priv->child_count,
-                           child_index, local->transaction.type);
+	local->transaction.failed_subvols[child_index] = 1;
+}
+
+
+
+        static gf_boolean_t
+afr_locals_overlap (afr_local_t *local1, afr_local_t *local2)
+{
+        uint64_t start1 = local1->transaction.start;
+        uint64_t start2 = local2->transaction.start;
+        uint64_t end1 = 0;
+        uint64_t end2 = 0;
+
+        if (local1->transaction.len)
+                end1 = start1 + local1->transaction.len - 1;
+        else
+                end1 = ULLONG_MAX;
+
+        if (local2->transaction.len)
+                end2 = start2 + local2->transaction.len - 1;
+        else
+                end2 = ULLONG_MAX;
+
+        return ((end1 >= start2) && (end2 >= start1));
+}
+
+void
+afr_transaction_eager_lock_init (afr_local_t *local, xlator_t *this)
+{
+        afr_private_t *priv = NULL;
+        afr_fd_ctx_t  *fdctx = NULL;
+        afr_local_t   *each = NULL;
+
+        priv = this->private;
+
+        if (!local->fd)
+                return;
+
+        if (local->transaction.type != AFR_DATA_TRANSACTION)
+                return;
+
+        if (!priv->eager_lock)
+                return;
+
+        fdctx = afr_fd_ctx_get (local->fd, this);
+        if (!fdctx)
+                return;
+
+        if (afr_are_multiple_fds_opened (local->fd, this))
+                return;
+        /*
+         * Once full file lock is acquired in eager-lock phase, overlapping
+         * writes do not compete for inode-locks, instead are transferred to the
+         * next writes. Because of this overlapping writes are not ordered.
+         * This can cause inconsistencies in replication.
+         * Example:
+         * Two overlapping writes w1, w2 are sent in parallel on same fd
+         * in two threads t1, t2.
+         * Both threads can execute afr_writev_wind in the following manner.
+         * t1 winds w1 on brick-0
+         * t2 winds w2 on brick-0
+         * t2 winds w2 on brick-1
+         * t1 winds w1 on brick-1
+         *
+         * This check makes sure the locks are not transferred for
+         * overlapping writes.
+         */
+        LOCK (&local->fd->lock);
+        {
+                list_for_each_entry (each, &fdctx->eager_locked,
+                                     transaction.eager_locked) {
+                        if (afr_locals_overlap (each, local)) {
+                                local->transaction.eager_lock_on = _gf_false;
+                                goto unlock;
+                        }
+                }
+
+                local->transaction.eager_lock_on = _gf_true;
+                list_add_tail (&local->transaction.eager_locked,
+                               &fdctx->eager_locked);
+        }
+unlock:
+        UNLOCK (&local->fd->lock);
 }
 
 
@@ -1275,20 +1680,43 @@ afr_transaction (call_frame_t *frame, xlator_t *this, afr_transaction_type type)
 {
         afr_local_t *   local = NULL;
         afr_private_t * priv  = NULL;
+        fd_t            *fd   = NULL;
+        int             ret   = -1;
 
         local = frame->local;
         priv  = this->private;
 
-        afr_transaction_local_init (local, this);
-
         local->transaction.resume = afr_transaction_resume;
         local->transaction.type   = type;
 
+        ret = afr_transaction_local_init (local, this);
+        if (ret < 0)
+            goto out;
+
+        afr_transaction_eager_lock_init (local, this);
+
+        if (local->fd && local->transaction.eager_lock_on)
+                afr_set_lk_owner (frame, this, local->fd);
+        else
+                afr_set_lk_owner (frame, this, frame->root);
+
+        if (!local->transaction.eager_lock_on && local->loc.inode) {
+                fd = fd_lookup (local->loc.inode, frame->root->pid);
+                if (fd == NULL)
+                        fd = fd_lookup_anonymous (local->loc.inode);
+
+                if (fd) {
+                        afr_delayed_changelog_wake_up (this, fd);
+                        fd_unref (fd);
+                }
+        }
+
         if (afr_lock_server_count (priv, local->transaction.type) == 0) {
                 afr_internal_lock_finish (frame, this);
         } else {
                 afr_lock (frame, this);
         }
-
-        return 0;
+        ret = 0;
+out:
+        return ret;
 }
diff --git a/xlators/cluster/afr/src/afr-transaction.h b/xlators/cluster/afr/src/afr-transaction.h
index 965ee6737..77cc8eed0 100644
--- a/xlators/cluster/afr/src/afr-transaction.h
+++ b/xlators/cluster/afr/src/afr-transaction.h
@@ -11,6 +11,8 @@
 #ifndef __TRANSACTION_H__
 #define __TRANSACTION_H__
 
+#include "afr.h"
+
 void
 afr_transaction_fop_failed (call_frame_t *frame, xlator_t *this,
 			    int child_index);
@@ -18,11 +20,34 @@ afr_transaction_fop_failed (call_frame_t *frame, xlator_t *this,
 int
 afr_lock_server_count (afr_private_t *priv, afr_transaction_type type);
 
+afr_inodelk_t*
+afr_get_inodelk (afr_internal_lock_t *int_lock, char *dom);
+
 int32_t
 afr_transaction (call_frame_t *frame, xlator_t *this, afr_transaction_type type);
 
-afr_fd_ctx_t *
-afr_fd_ctx_get (fd_t *fd, xlator_t *this);
 int
 afr_set_pending_dict (afr_private_t *priv, dict_t *xattr, int32_t **pending);
+
+void
+afr_set_delayed_post_op (call_frame_t *frame, xlator_t *this);
+
+void
+afr_delayed_changelog_wake_up (xlator_t *this, fd_t *fd);
+
+void
+__mark_all_success (call_frame_t *frame, xlator_t *this);
+
+gf_boolean_t
+afr_txn_nothing_failed (call_frame_t *frame, xlator_t *this);
+
+int afr_read_txn (call_frame_t *frame, xlator_t *this, inode_t *inode,
+		  afr_read_txn_wind_t readfn, afr_transaction_type type);
+
+int afr_read_txn_continue (call_frame_t *frame, xlator_t *this, int subvol);
+
+int __afr_txn_write_fop (call_frame_t *frame, xlator_t *this);
+int __afr_txn_write_done (call_frame_t *frame, xlator_t *this);
+call_frame_t *afr_transaction_detach_fop_frame (call_frame_t *frame);
+
 #endif /* __TRANSACTION_H__ */
diff --git a/xlators/cluster/afr/src/afr.c b/xlators/cluster/afr/src/afr.c
index 3f102f808..5e12910b7 100644
--- a/xlators/cluster/afr/src/afr.c
+++ b/xlators/cluster/afr/src/afr.c
@@ -21,11 +21,6 @@
 #endif
 #include "afr-common.c"
 
-#define SHD_INODE_LRU_LIMIT          2048
-#define AFR_EH_HEALED_LIMIT          1024
-#define AFR_EH_HEAL_FAIL_LIMIT       1024
-#define AFR_EH_SPLIT_BRAIN_LIMIT     1024
-
 struct volume_options options[];
 
 int32_t
@@ -107,12 +102,21 @@ reconfigure (xlator_t *this, dict_t *options)
 {
         afr_private_t *priv        = NULL;
         xlator_t      *read_subvol = NULL;
+        int            read_subvol_index = -1;
         int            ret         = -1;
         int            index       = -1;
         char          *qtype       = NULL;
 
         priv = this->private;
 
+	GF_OPTION_RECONF ("afr-dirty-xattr",
+			  priv->afr_dirty, options, str,
+			  out);
+
+	GF_OPTION_RECONF ("metadata-splitbrain-forced-heal",
+			  priv->metadata_splitbrain_forced_heal, options, bool,
+			  out);
+
         GF_OPTION_RECONF ("background-self-heal-count",
                           priv->background_self_heal_count, options, uint32,
                           out);
@@ -126,9 +130,6 @@ reconfigure (xlator_t *this, dict_t *options)
         GF_OPTION_RECONF ("entry-self-heal", priv->entry_self_heal, options,
                           bool, out);
 
-        GF_OPTION_RECONF ("strict-readdir", priv->strict_readdir, options, bool,
-                          out);
-
         GF_OPTION_RECONF ("data-self-heal-window-size",
                           priv->data_self_heal_window_size, options,
                           uint32, out);
@@ -145,10 +146,11 @@ reconfigure (xlator_t *this, dict_t *options)
         GF_OPTION_RECONF ("data-self-heal-algorithm",
                           priv->data_self_heal_algorithm, options, str, out);
 
-        GF_OPTION_RECONF ("self-heal-daemon", priv->shd.enabled, options, bool, out);
-
         GF_OPTION_RECONF ("read-subvolume", read_subvol, options, xlator, out);
 
+        GF_OPTION_RECONF ("read-hash-mode", priv->hash_mode,
+                          options, uint32, out);
+
         if (read_subvol) {
                 index = xlator_subvolume_index (this, read_subvol);
                 if (index == -1) {
@@ -159,12 +161,43 @@ reconfigure (xlator_t *this, dict_t *options)
                 priv->read_child = index;
         }
 
+        GF_OPTION_RECONF ("read-subvolume-index",read_subvol_index, options,int32,out);
+
+        if (read_subvol_index >-1) {
+                index=read_subvol_index;
+                if (index >= priv->child_count) {
+                        gf_log (this->name, GF_LOG_ERROR, "%d not a subvolume-index",
+                                index);
+                        goto out;
+                }
+                priv->read_child = index;
+        }
+
+        GF_OPTION_RECONF ("pre-op-compat", priv->pre_op_compat, options, bool, out);
+
         GF_OPTION_RECONF ("eager-lock", priv->eager_lock, options, bool, out);
         GF_OPTION_RECONF ("quorum-type", qtype, options, str, out);
         GF_OPTION_RECONF ("quorum-count", priv->quorum_count, options,
                           uint32, out);
         fix_quorum_options(this,priv,qtype);
 
+	GF_OPTION_RECONF ("post-op-delay-secs", priv->post_op_delay_secs, options,
+			  uint32, out);
+
+        GF_OPTION_RECONF (AFR_SH_READDIR_SIZE_KEY, priv->sh_readdir_size,
+                          options, size, out);
+        /* Reset this so we re-discover in case the topology changed.  */
+        GF_OPTION_RECONF ("ensure-durability", priv->ensure_durability, options,
+                          bool, out);
+
+	GF_OPTION_RECONF ("self-heal-daemon", priv->shd.enabled, options,
+			  bool, out);
+
+	GF_OPTION_RECONF ("iam-self-heal-daemon", priv->shd.iamshd, options,
+			  bool, out);
+
+        priv->did_discovery = _gf_false;
+
         ret = 0;
 out:
         return ret;
@@ -191,6 +224,7 @@ init (xlator_t *this)
         int            ret         = -1;
         GF_UNUSED int  op_errno    = 0;
         xlator_t      *read_subvol = NULL;
+        int            read_subvol_index = -1;
         xlator_t      *fav_child   = NULL;
         char          *qtype       = NULL;
 
@@ -213,18 +247,18 @@ init (xlator_t *this)
 
         priv = this->private;
         LOCK_INIT (&priv->lock);
-        LOCK_INIT (&priv->read_child_lock);
-        //lock recovery is not done in afr
-        pthread_mutex_init (&priv->mutex, NULL);
-        INIT_LIST_HEAD (&priv->saved_fds);
 
         child_count = xlator_subvolume_count (this);
 
         priv->child_count = child_count;
 
-
         priv->read_child = -1;
 
+	GF_OPTION_INIT ("afr-dirty-xattr", priv->afr_dirty, str, out);
+
+	GF_OPTION_INIT ("metadata-splitbrain-forced-heal",
+			priv->metadata_splitbrain_forced_heal, bool, out);
+
         GF_OPTION_INIT ("read-subvolume", read_subvol, xlator, out);
         if (read_subvol) {
                 priv->read_child = xlator_subvolume_index (this, read_subvol);
@@ -234,6 +268,18 @@ init (xlator_t *this)
                         goto out;
                 }
         }
+        GF_OPTION_INIT ("read-subvolume-index",read_subvol_index,int32,out);
+        if (read_subvol_index > -1) {
+                if (read_subvol_index >= priv->child_count) {
+                        gf_log (this->name, GF_LOG_ERROR, "%d not a subvolume-index",
+                                read_subvol_index);
+                        goto out;
+                }
+                priv->read_child = read_subvol_index;
+        }
+        GF_OPTION_INIT ("choose-local", priv->choose_local, bool, out);
+
+        GF_OPTION_INIT ("read-hash-mode", priv->hash_mode, uint32, out);
 
         priv->favorite_child = -1;
         GF_OPTION_INIT ("favorite-child", fav_child, xlator, out);
@@ -266,10 +312,6 @@ init (xlator_t *this)
 
         GF_OPTION_INIT ("entry-self-heal", priv->entry_self_heal, bool, out);
 
-        GF_OPTION_INIT ("self-heal-daemon", priv->shd.enabled, bool, out);
-
-        GF_OPTION_INIT ("iam-self-heal-daemon", priv->shd.iamshd, bool, out);
-
         GF_OPTION_INIT ("data-change-log", priv->data_change_log, bool, out);
 
         GF_OPTION_INIT ("metadata-change-log", priv->metadata_change_log, bool,
@@ -284,13 +326,23 @@ init (xlator_t *this)
 
         GF_OPTION_INIT ("entrylk-trace", priv->entrylk_trace, bool, out);
 
-        GF_OPTION_INIT ("strict-readdir", priv->strict_readdir, bool, out);
+        GF_OPTION_INIT ("pre-op-compat", priv->pre_op_compat, bool, out);
 
         GF_OPTION_INIT ("eager-lock", priv->eager_lock, bool, out);
         GF_OPTION_INIT ("quorum-type", qtype, str, out);
         GF_OPTION_INIT ("quorum-count", priv->quorum_count, uint32, out);
+        GF_OPTION_INIT (AFR_SH_READDIR_SIZE_KEY, priv->sh_readdir_size, size,
+                        out);
         fix_quorum_options(this,priv,qtype);
 
+	GF_OPTION_INIT ("post-op-delay-secs", priv->post_op_delay_secs, uint32, out);
+        GF_OPTION_INIT ("ensure-durability", priv->ensure_durability, bool,
+                        out);
+
+	GF_OPTION_INIT ("self-heal-daemon", priv->shd.enabled, bool, out);
+
+	GF_OPTION_INIT ("iam-self-heal-daemon", priv->shd.iamshd, bool, out);
+
         priv->wait_count = 1;
 
         priv->child_up = GF_CALLOC (sizeof (unsigned char), child_count,
@@ -331,8 +383,6 @@ init (xlator_t *this)
                                    AFR_XATTR_PREFIX,
                                    trav->xlator->name);
                 if (-1 == ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "asprintf failed to set pending key");
                         ret = -ENOMEM;
                         goto out;
                 }
@@ -341,6 +391,13 @@ init (xlator_t *this)
                 i++;
         }
 
+        ret = gf_asprintf (&priv->sh_domain, AFR_SH_DATA_DOMAIN_FMT,
+                           this->name);
+        if (-1 == ret) {
+                ret = -ENOMEM;
+                goto out;
+        }
+
         priv->last_event = GF_CALLOC (child_count, sizeof (*priv->last_event),
                                       gf_afr_mt_int32_t);
         if (!priv->last_event) {
@@ -348,6 +405,12 @@ init (xlator_t *this)
                 goto out;
         }
 
+	ret = afr_selfheal_daemon_init (this);
+	if (ret) {
+		ret = -ENOMEM;
+		goto out;
+	}
+
         /* keep more local here as we may need them for self-heal etc */
         this->local_pool = mem_pool_new (afr_local_t, 512);
         if (!this->local_pool) {
@@ -357,52 +420,8 @@ init (xlator_t *this)
                 goto out;
         }
 
-        priv->first_lookup = 1;
         priv->root_inode = NULL;
 
-        if (!priv->shd.iamshd) {
-                ret = 0;
-                goto out;
-        }
-
-        ret = -ENOMEM;
-        priv->shd.pos = GF_CALLOC (sizeof (*priv->shd.pos), child_count,
-                                   gf_afr_mt_brick_pos_t);
-        if (!priv->shd.pos)
-                goto out;
-
-        priv->shd.pending = GF_CALLOC (sizeof (*priv->shd.pending), child_count,
-                                       gf_afr_mt_int32_t);
-        if (!priv->shd.pending)
-                goto out;
-
-        priv->shd.inprogress = GF_CALLOC (sizeof (*priv->shd.inprogress),
-                                          child_count, gf_afr_mt_shd_bool_t);
-        if (!priv->shd.inprogress)
-                goto out;
-        priv->shd.timer = GF_CALLOC (sizeof (*priv->shd.timer), child_count,
-                                     gf_afr_mt_shd_timer_t);
-        if (!priv->shd.timer)
-                goto out;
-
-        priv->shd.healed = eh_new (AFR_EH_HEALED_LIMIT, _gf_false);
-        if (!priv->shd.healed)
-                goto out;
-
-        priv->shd.heal_failed = eh_new (AFR_EH_HEAL_FAIL_LIMIT, _gf_false);
-        if (!priv->shd.heal_failed)
-                goto out;
-
-        priv->shd.split_brain = eh_new (AFR_EH_SPLIT_BRAIN_LIMIT, _gf_false);
-        if (!priv->shd.split_brain)
-                goto out;
-
-        this->itable = inode_table_new (SHD_INODE_LRU_LIMIT, this);
-        if (!this->itable)
-                goto out;
-        priv->root_inode = inode_ref (this->itable->root);
-        GF_OPTION_INIT ("node-uuid", priv->shd.node_uuid, str, out);
-
         ret = 0;
 out:
         return ret;
@@ -437,6 +456,9 @@ struct xlator_fops fops = {
         .finodelk    = afr_finodelk,
         .entrylk     = afr_entrylk,
         .fentrylk    = afr_fentrylk,
+	.fallocate   = afr_fallocate,
+	.discard     = afr_discard,
+        .zerofill    = afr_zerofill,
 
         /* inode read */
         .access      = afr_access,
@@ -489,33 +511,79 @@ struct xlator_cbks cbks = {
 
 struct volume_options options[] = {
         { .key  = {"read-subvolume" },
-          .type = GF_OPTION_TYPE_XLATOR
+          .type = GF_OPTION_TYPE_XLATOR,
+          .description = "inode-read fops happen only on one of the bricks in "
+                         "replicate. Afr will prefer the one specified using "
+                         "this option if it is not stale. Option value must be "
+                         "one of the xlator names of the children. "
+                         "Ex: <volname>-client-0 till "
+                         "<volname>-client-<number-of-bricks - 1>"
+        },
+        { .key  = {"read-subvolume-index" },
+          .type = GF_OPTION_TYPE_INT,
+          .default_value = "-1",
+          .description = "inode-read fops happen only on one of the bricks in "
+                         "replicate. AFR will prefer the one specified using "
+                         "this option if it is not stale. allowed options"
+                         " include -1 till replica-count - 1"
+        },
+        { .key = {"read-hash-mode" },
+          .type = GF_OPTION_TYPE_INT,
+          .min = 0,
+          .max = 2,
+          .default_value = "1",
+          .description = "inode-read fops happen only on one of the bricks in "
+                         "replicate. AFR will prefer the one computed using "
+                         "the method specified using this option"
+                         "0 = first up server, "
+                         "1 = hash by GFID of file (all clients use "
+                                                    "same subvolume), "
+                         "2 = hash by GFID of file and client PID",
+        },
+        { .key  = {"choose-local" },
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "true",
+          .description = "Choose a local subvolume (i.e. Brick) to read from"
+	                 " if read-subvolume is not explicitly set.",
         },
         { .key  = {"favorite-child"},
-          .type = GF_OPTION_TYPE_XLATOR
+          .type = GF_OPTION_TYPE_XLATOR,
+          .description = "If a split-brain happens choose subvol/brick set by "
+                         "this option as source."
         },
         { .key  = {"background-self-heal-count"},
           .type = GF_OPTION_TYPE_INT,
           .min  = 0,
           .default_value = "16",
+          .validate = GF_OPT_VALIDATE_MIN,
+          .description = "This specifies the number of self-heals that can be "
+                         " performed in background without blocking the fop"
         },
         { .key  = {"data-self-heal"},
           .type = GF_OPTION_TYPE_STR,
-          .default_value = "",
           .value = {"1", "on", "yes", "true", "enable",
                     "0", "off", "no", "false", "disable",
                     "open"},
           .default_value = "on",
+          .description   = "Using this option we can enable/disable data "
+                           "self-heal on the file. \"open\" means data "
+                           "self-heal action will only be triggered by file "
+                           "open operations."
         },
         { .key  = {"data-self-heal-algorithm"},
           .type = GF_OPTION_TYPE_STR,
-          .default_value = "",
           .description   = "Select between \"full\", \"diff\". The "
                            "\"full\" algorithm copies the entire file from "
                            "source to sink. The \"diff\" algorithm copies to "
                            "sink only those blocks whose checksums don't match "
-                           "with those of source.",
-          .value = { "diff", "full", "" }
+                           "with those of source. If no option is configured "
+                           "the option is chosen dynamically as follows: "
+                           "If the file does not exist on one of the sinks "
+                           "or empty file exists or if the source file size is "
+                           "about the same as page size the entire file will "
+                           "be read and written i.e \"full\" algo, "
+                           "otherwise \"diff\" algo is chosen.",
+          .value = { "diff", "full"}
         },
         { .key  = {"data-self-heal-window-size"},
           .type = GF_OPTION_TYPE_INT,
@@ -528,54 +596,102 @@ struct volume_options options[] = {
         { .key  = {"metadata-self-heal"},
           .type = GF_OPTION_TYPE_BOOL,
           .default_value = "on",
+          .description = "Using this option we can enable/disable metadata "
+                         "i.e. Permissions, ownerships, xattrs self-heal on "
+                         "the file/directory."
         },
         { .key  = {"entry-self-heal"},
           .type = GF_OPTION_TYPE_BOOL,
           .default_value = "on",
+          .description = "Using this option we can enable/disable entry "
+                         "self-heal on the directory."
         },
         { .key  = {"data-change-log"},
           .type = GF_OPTION_TYPE_BOOL,
           .default_value = "on",
+          .description = "Data fops like write/truncate will not perform "
+                         "pre/post fop changelog operations in afr transaction "
+                         "if this option is disabled"
         },
         { .key  = {"metadata-change-log"},
           .type = GF_OPTION_TYPE_BOOL,
           .default_value = "on",
+          .description = "Metadata fops like setattr/setxattr will not perform "
+                         "pre/post fop changelog operations in afr transaction "
+                         "if this option is disabled"
         },
         { .key  = {"entry-change-log"},
           .type = GF_OPTION_TYPE_BOOL,
           .default_value = "on",
+          .description = "Entry fops like create/unlink will not perform "
+                         "pre/post fop changelog operations in afr transaction "
+                         "if this option is disabled"
         },
         { .key  = {"optimistic-change-log"},
           .type = GF_OPTION_TYPE_BOOL,
           .default_value = "on",
-        },
-        { .key  = {"strict-readdir"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
+          .description = "Entry/Metadata fops will not perform "
+                         "pre fop changelog operations in afr transaction "
+                         "if this option is enabled."
         },
         { .key = {"inodelk-trace"},
           .type = GF_OPTION_TYPE_BOOL,
           .default_value = "off",
+          .description = "Enabling this option logs inode lock/unlocks"
         },
         { .key = {"entrylk-trace"},
           .type = GF_OPTION_TYPE_BOOL,
           .default_value = "off",
+          .description = "Enabling this option logs entry lock/unlocks"
         },
+	{ .key = {"pre-op-compat"},
+	  .type = GF_OPTION_TYPE_BOOL,
+	  .default_value = "on",
+	  .description = "Use separate pre-op xattrop() FOP rather than "
+	                 "overloading xdata of the OP"
+	},
         { .key = {"eager-lock"},
           .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
+          .default_value = "on",
+          .description = "Lock phase of a transaction has two sub-phases. "
+                         "First is an attempt to acquire locks in parallel by "
+                         "broadcasting non-blocking lock requests. If lock "
+                         "acquisition fails on any server, then the held locks "
+                         "are unlocked and revert to a blocking locked mode "
+                         "sequentially on one server after another.  If this "
+                         "option is enabled the initial broadcasting lock "
+                         "request attempt to acquire lock on the entire file. "
+                         "If this fails, we revert back to the sequential "
+                         "\"regional\" blocking lock as before. In the case "
+                         "where such an \"eager\" lock is granted in the "
+                         "non-blocking phase, it gives rise to an opportunity "
+                         "for optimization. i.e, if the next write transaction "
+                         "on the same FD arrives before the unlock phase of "
+                         "the first transaction, it \"takes over\" the full "
+                         "file lock. Similarly if yet another data transaction "
+                         "arrives before the unlock phase of the \"optimized\" "
+                         "transaction, that in turn \"takes over\" the lock as "
+                         "well. The actual unlock now happens at the end of "
+                         "the last \"optimized\" transaction."
+
         },
         { .key = {"self-heal-daemon"},
           .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
+          .default_value = "on",
+          .description = "This option applies to only self-heal-daemon. "
+                         "Index directory crawl and automatic healing of files "
+                         "will not be performed if this option is turned off."
         },
         { .key = {"iam-self-heal-daemon"},
           .type = GF_OPTION_TYPE_BOOL,
           .default_value = "off",
+          .description = "This option differentiates if the replicate "
+                         "translator is running as part of self-heal-daemon "
+                         "or not."
         },
         { .key = {"quorum-type"},
           .type = GF_OPTION_TYPE_STR,
-          .value = { "none", "auto", "fixed", "" },
+          .value = { "none", "auto", "fixed"},
           .default_value = "none",
           .description = "If value is \"fixed\" only allow writes if "
                          "quorum-count bricks are present.  If value is "
@@ -594,7 +710,40 @@ struct volume_options options[] = {
         },
         { .key  = {"node-uuid"},
           .type = GF_OPTION_TYPE_STR,
-          .description = "Local glusterd uuid string",
+          .description = "Local glusterd uuid string, used in starting "
+                         "self-heal-daemon so that it can crawl only on "
+                         "local index directories.",
+        },
+        { .key  = {"post-op-delay-secs"},
+          .type = GF_OPTION_TYPE_INT,
+          .min  = 0,
+          .max  = INT_MAX,
+          .default_value = "1",
+          .description = "Time interval induced artificially before "
+	                 "post-operation phase of the transaction to "
+                         "enhance overlap of adjacent write operations.",
+        },
+        { .key = {AFR_SH_READDIR_SIZE_KEY},
+          .type = GF_OPTION_TYPE_SIZET,
+          .description = "readdirp size for performing entry self-heal",
+          .min = 1024,
+          .max = 131072,
+          .default_value = "1KB",
+        },
+        { .key = {"ensure-durability"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .description = "Afr performs fsyncs for transactions if this "
+                         "option is on to make sure the changelogs/data is "
+                         "written to the disk",
+          .default_value = "on",
         },
+	{ .key = {"afr-dirty-xattr"},
+	  .type = GF_OPTION_TYPE_STR,
+	  .default_value = AFR_DIRTY_DEFAULT,
+	},
+	{ .key = {"metadata-splitbrain-forced-heal"},
+	  .type = GF_OPTION_TYPE_BOOL,
+	  .default_value = "off",
+	},
         { .key  = {NULL} },
 };
diff --git a/xlators/cluster/afr/src/afr.h b/xlators/cluster/afr/src/afr.h
index fe0e2b2e6..36042f7b2 100644
--- a/xlators/cluster/afr/src/afr.h
+++ b/xlators/cluster/afr/src/afr.h
@@ -20,94 +20,42 @@
 #include "call-stub.h"
 #include "compat-errno.h"
 #include "afr-mem-types.h"
-#include "afr-self-heal-algorithm.h"
 
 #include "libxlator.h"
 #include "timer.h"
+#include "syncop.h"
+
+#include "afr-self-heald.h"
 
 #define AFR_XATTR_PREFIX "trusted.afr"
 #define AFR_PATHINFO_HEADER "REPLICATE:"
+#define AFR_SH_READDIR_SIZE_KEY "self-heal-readdir-size"
+#define AFR_SH_DATA_DOMAIN_FMT "%s:self-heal"
+#define AFR_DIRTY_DEFAULT AFR_XATTR_PREFIX ".dirty"
+#define AFR_DIRTY (((afr_private_t *) (THIS->private))->afr_dirty)
 
-struct _pump_private;
-
-typedef int (*afr_expunge_done_cbk_t) (call_frame_t *frame, xlator_t *this,
-                                       int child, int32_t op_error,
-                                       int32_t op_errno);
-
-typedef int (*afr_impunge_done_cbk_t) (call_frame_t *frame, xlator_t *this,
-                                       int32_t op_error, int32_t op_errno);
-typedef int (*afr_post_remove_call_t) (call_frame_t *frame, xlator_t *this);
+#define AFR_LOCKEE_COUNT_MAX    3
+#define AFR_DOM_COUNT_MAX    3
+#define AFR_NUM_CHANGE_LOGS            3 /*data + metadata + entry*/
 
 typedef int (*afr_lock_cbk_t) (call_frame_t *frame, xlator_t *this);
-typedef void (*afr_lookup_done_cbk_t) (call_frame_t *frame, xlator_t *this,
-                                      int32_t op_ret, int32_t op_errno);
 
-typedef enum {
-        AFR_POS_UNKNOWN,
-        AFR_POS_LOCAL,
-        AFR_POS_REMOTE
-} afr_child_pos_t;
+typedef int (*afr_read_txn_wind_t) (call_frame_t *frame, xlator_t *this, int subvol);
 
-typedef enum {
-        SPLIT_BRAIN = 1,
-        ALL_FOOLS = 2
-} afr_subvol_status_t;
+typedef int (*afr_inode_refresh_cbk_t) (call_frame_t *frame, xlator_t *this, int err);
 
-typedef enum {
-        AFR_INODE_SET_READ_CTX = 1,
-        AFR_INODE_RM_STALE_CHILDREN,
-        AFR_INODE_SET_OPENDIR_DONE,
-        AFR_INODE_SET_SPLIT_BRAIN,
-        AFR_INODE_GET_READ_CTX,
-        AFR_INODE_GET_OPENDIR_DONE,
-        AFR_INODE_GET_SPLIT_BRAIN,
-} afr_inode_op_t;
-
-typedef struct afr_inode_params_ {
-        afr_inode_op_t op;
-        union {
-                gf_boolean_t value;
-                struct {
-                        int32_t read_child;
-                        int32_t *children;
-                } read_ctx;
-        } u;
-} afr_inode_params_t;
+typedef int (*afr_changelog_resume_t) (call_frame_t *frame, xlator_t *this);
 
-typedef struct afr_inode_ctx_ {
-        uint64_t masks;
-        int32_t  *fresh_children;//increasing order of latency
-} afr_inode_ctx_t;
-
-typedef enum {
-        NONE,
-        INDEX,
-        FULL,
-} afr_crawl_type_t;
-
-typedef struct afr_self_heald_ {
-        gf_boolean_t     enabled;
-        gf_boolean_t     iamshd;
-        afr_crawl_type_t *pending;
-        gf_boolean_t     *inprogress;
-        afr_child_pos_t  *pos;
-        gf_timer_t       **timer;
-        eh_t             *healed;
-        eh_t             *heal_failed;
-        eh_t             *split_brain;
-        char             *node_uuid;
-} afr_self_heald_t;
+#define alloca0(size) ({void *__ptr; __ptr = alloca(size); memset(__ptr, 0, size); __ptr;})
+#define AFR_COUNT(array,max) ({int __i; int __res = 0; for (__i = 0; __i < max; __i++) if (array[__i]) __res++; __res;})
+#define AFR_INTERSECT(dst,src1,src2,max) ({int __i; for (__i = 0; __i < max; __i++) dst[__i] = src1[__i] && src2[__i];})
 
 typedef struct _afr_private {
         gf_lock_t lock;               /* to guard access to child_count, etc */
         unsigned int child_count;     /* total number of children   */
 
-        unsigned int read_child_rr;   /* round-robin index of the read_child */
-        gf_lock_t read_child_lock;    /* lock to protect above */
-
         xlator_t **children;
 
-        int first_lookup;
         inode_t *root_inode;
 
         unsigned char *child_up;
@@ -128,144 +76,54 @@ typedef struct _afr_private {
         gf_boolean_t metadata_change_log;   /* on/off */
         gf_boolean_t entry_change_log;      /* on/off */
 
+	gf_boolean_t metadata_splitbrain_forced_heal; /* on/off */
         int read_child;               /* read-subvolume */
+        unsigned int hash_mode;       /* for when read_child is not set */
         int favorite_child;  /* subvolume to be preferred in resolving
                                          split-brain cases */
 
         gf_boolean_t inodelk_trace;
         gf_boolean_t entrylk_trace;
 
-        gf_boolean_t strict_readdir;
-
         unsigned int wait_count;      /* # of servers to wait for success */
 
         uint64_t up_count;      /* number of CHILD_UPs we have seen */
         uint64_t down_count;    /* number of CHILD_DOWNs we have seen */
 
-        struct _pump_private *pump_private; /* Set if we are loaded as pump */
-        int                   use_afr_in_pump;
-
-        pthread_mutex_t  mutex;
-        struct list_head saved_fds;   /* list of fds on which locks have succeeded */
         gf_boolean_t      optimistic_change_log;
         gf_boolean_t      eager_lock;
+        gf_boolean_t      pre_op_compat;      /* on/off */
+	uint32_t          post_op_delay_secs;
         unsigned int      quorum_count;
 
         char                   vol_uuid[UUID_SIZE + 1];
         int32_t                *last_event;
-        afr_self_heald_t       shd;
-} afr_private_t;
-
-typedef struct {
-        /* External interface: These are variables (some optional) that
-           are set by whoever has triggered self-heal */
-
-        gf_boolean_t do_data_self_heal;
-        gf_boolean_t do_metadata_self_heal;
-        gf_boolean_t do_entry_self_heal;
-        gf_boolean_t do_gfid_self_heal;
-        gf_boolean_t do_missing_entry_self_heal;
-
-        gf_boolean_t forced_merge;        /* Is this a self-heal triggered to
-                                             forcibly merge the directories? */
-
-        gf_boolean_t background;          /* do self-heal in background
-                                             if possible */
-        ia_type_t type;                   /* st_mode of the entry we're doing
-                                             self-heal on */
-        inode_t   *inode;                 /* inode on which the self-heal is
-                                             performed on */
-        uuid_t  sh_gfid_req;                 /* gfid self-heal needs to be done
-                                             with this gfid if it is not null */
-
-        /* Function to call to unwind. If self-heal is being done in the
-           background, this function will be called as soon as possible. */
-
-        int (*unwind) (call_frame_t *frame, xlator_t *this, int32_t op_ret,
-                       int32_t op_errno, int32_t sh_failed);
-
-        /* End of external interface members */
-
-
-        /* array of stat's, one for each child */
-        struct iatt *buf;
-        struct iatt *parentbufs;
-        struct iatt parentbuf;
-        struct iatt entrybuf;
-
-        afr_expunge_done_cbk_t expunge_done;
-        afr_impunge_done_cbk_t impunge_done;
-
-        /* array of xattr's, one for each child */
-        dict_t **xattr;
-
-        /* array containing if the lookups succeeded in the order of response
-         */
-        int32_t *success_children;
-        int     success_count;
-        /* array containing the fresh children found in the self-heal process */
-        int32_t *fresh_children;
-        /* array containing the fresh children found in the parent lookup */
-        int32_t *fresh_parent_dirs;
-        /* array of errno's, one for each child */
-        int *child_errno;
-        /*loc used for lookup*/
-        loc_t lookup_loc;
-        int32_t lookup_flags;
-        afr_lookup_done_cbk_t lookup_done;
-
-        int32_t **pending_matrix;
-        int32_t **delta_matrix;
 
-        int32_t op_ret;
-        int32_t op_errno;
+	/* @event_generation: Keeps count of number of events received which can
+	   potentially impact consistency decisions. The events are CHILD_UP
+	   and CHILD_DOWN, when we have to recalculate the freshness/staleness
+	   of copies to detect if changes had happened while the other server
+	   was down. CHILD_DOWN and CHILD_UP can also be received on network
+	   disconnect/reconnects and not necessarily server going down/up.
+	   Recalculating freshness/staleness on network events is equally
+	   important as we might have had a network split brain.
+	*/
+	uint32_t               event_generation;
+
+        gf_boolean_t           choose_local;
+        gf_boolean_t           did_discovery;
+        uint64_t               sh_readdir_size;
+        gf_boolean_t           ensure_durability;
+        char                   *sh_domain;
+	char                   *afr_dirty;
+
+	afr_self_heald_t       shd;
+
+	/* pump dependencies */
+	void                   *pump_private;
+	gf_boolean_t           use_afr_in_pump;
+} afr_private_t;
 
-        int *sources;
-        int source;
-        int active_source;
-        int active_sinks;
-        unsigned char *success;
-        unsigned char *locked_nodes;
-        int lock_count;
-
-        const char *linkname;
-        gf_boolean_t entries_skipped;
-
-        int   op_failed;
-
-        gf_boolean_t sync_done;
-        gf_boolean_t data_lock_held;
-        gf_boolean_t eof_reached;
-        fd_t  *healing_fd;
-        int   file_has_holes;
-        blksize_t block_size;
-        off_t file_size;
-        off_t offset;
-        unsigned char *write_needed;
-        uint8_t *checksum;
-        afr_post_remove_call_t post_remove_call;
-
-        loc_t parent_loc;
-
-        call_frame_t *orig_frame;
-        call_frame_t *old_loop_frame;
-        gf_boolean_t unwound;
-
-        afr_sh_algo_private_t *private;
-
-        struct afr_sh_algorithm  *algo;
-        afr_lock_cbk_t data_lock_success_handler;
-        afr_lock_cbk_t data_lock_failure_handler;
-        int (*completion_cbk) (call_frame_t *frame, xlator_t *this);
-        int (*sh_data_algo_start) (call_frame_t *frame, xlator_t *this);
-        int (*algo_completion_cbk) (call_frame_t *frame, xlator_t *this);
-        int (*algo_abort_cbk) (call_frame_t *frame, xlator_t *this);
-        void (*gfid_sh_success_cbk) (call_frame_t *sh_frame, xlator_t *this);
-        gf_boolean_t mdata_spb;
-        gf_boolean_t data_spb;
-
-        call_frame_t *sh_frame;
-} afr_self_heal_t;
 
 typedef enum {
         AFR_DATA_TRANSACTION,          /* truncate, write, ... */
@@ -327,11 +185,31 @@ afr_index_for_transaction_type (afr_transaction_type type)
         return -1;  /* make gcc happy */
 }
 
+typedef struct {
+        loc_t                   loc;
+        char                    *basename;
+        unsigned char           *locked_nodes;
+        int                     locked_count;
+
+} afr_entry_lockee_t;
+
+int
+afr_entry_lockee_cmp (const void *l1, const void *l2);
+
+typedef struct {
+        char    *domain; /* Domain on which inodelk is taken */
+        struct gf_flock flock;
+        unsigned char *locked_nodes;
+        int32_t lock_count;
+} afr_inodelk_t;
 
 typedef struct {
         loc_t *lk_loc;
-        struct gf_flock lk_flock;
 
+        int                     lockee_count;
+        afr_entry_lockee_t      lockee[AFR_LOCKEE_COUNT_MAX];
+
+        afr_inodelk_t       inodelk[AFR_DOM_COUNT_MAX];
         const char *lk_basename;
         const char *lower_basename;
         const char *higher_basename;
@@ -340,84 +218,203 @@ typedef struct {
 
         unsigned char *locked_nodes;
         unsigned char *lower_locked_nodes;
-        unsigned char *inode_locked_nodes;
-        unsigned char *entry_locked_nodes;
 
         selfheal_lk_type_t selfheal_lk_type;
         transaction_lk_type_t transaction_lk_type;
 
         int32_t lock_count;
-        int32_t inodelk_lock_count;
         int32_t entrylk_lock_count;
 
         uint64_t lock_number;
         int32_t lk_call_count;
         int32_t lk_expected_count;
+        int32_t lk_attempted_count;
 
         int32_t lock_op_ret;
         int32_t lock_op_errno;
         afr_lock_cbk_t lock_cbk;
+        char *domain; /* Domain on which inode/entry lock/unlock in progress.*/
 } afr_internal_lock_t;
 
-typedef struct _afr_locked_fd {
-        fd_t  *fd;
-        struct list_head list;
-} afr_locked_fd_t;
+struct afr_reply {
+	int	valid;
+	int32_t	op_ret;
+	int32_t	op_errno;
+	dict_t *xdata;
+	struct iatt poststat;
+	struct iatt postparent;
+	struct iatt prestat;
+	struct iatt preparent;
+	struct iatt preparent2;
+	struct iatt postparent2;
+	uint8_t checksum[MD5_DIGEST_LENGTH];
+};
+
+typedef enum {
+        AFR_FD_NOT_OPENED,
+        AFR_FD_OPENED,
+        AFR_FD_OPENING
+} afr_fd_open_status_t;
+
+typedef struct {
+        unsigned int *pre_op_done[AFR_NUM_CHANGE_LOGS];
+	int inherited[AFR_NUM_CHANGE_LOGS];
+	int on_disk[AFR_NUM_CHANGE_LOGS];
+        afr_fd_open_status_t *opened_on; /* which subvolumes the fd is open on */
+
+        unsigned int *lock_piggyback;
+        unsigned int *lock_acquired;
+
+        int flags;
+
+	/* used for delayed-post-op optimization */
+	pthread_mutex_t    delay_lock;
+	gf_timer_t        *delay_timer;
+	call_frame_t      *delay_frame;
+
+	/* set if any write on this fd was a non stable write
+	   (i.e, without O_SYNC or O_DSYNC)
+	*/
+	gf_boolean_t      witnessed_unstable_write;
+
+	/* @open_fd_count:
+	   Number of open FDs queried from the server, as queried through
+	   xdata in FOPs. Currently, used to decide if eager-locking must be
+	   temporarily disabled.
+	*/
+        uint32_t        open_fd_count;
+
+
+	/* list of frames currently in progress */
+	struct list_head  eager_locked;
+} afr_fd_ctx_t;
+
 
 typedef struct _afr_local {
-        int     uid;
-        int     gid;
+	glusterfs_fop_t  op;
         unsigned int call_count;
-        unsigned int success_count;
-        unsigned int enoent_count;
-        unsigned int govinda_gOvinda;
-        unsigned int read_child_index;
-        unsigned char read_child_returned;
-        unsigned int first_up_child;
 
-        pid_t saved_pid;
+	/* @event_generation: copy of priv->event_generation taken at the
+	   time of starting the transaction. The copy is made so that we
+	   have a stable value through the various phases of the transaction.
+	*/
+	unsigned int event_generation;
+
+        uint32_t     open_fd_count;
+        gf_boolean_t update_open_fd_count;
+
+	gf_lkowner_t  saved_lk_owner;
 
         int32_t op_ret;
         int32_t op_errno;
 
         int32_t **pending;
 
+	int dirty[AFR_NUM_CHANGE_LOGS];
+
         loc_t loc;
         loc_t newloc;
 
         fd_t *fd;
-        unsigned char *fd_open_on;
-
-        glusterfs_fop_t fop;
+	afr_fd_ctx_t *fd_ctx;
 
+	/* @child_up: copy of priv->child_up taken at the time of transaction
+	   start. The copy is taken so that we have a stable child_up array
+	   through the phases of the transaction as priv->child_up[i] can keep
+	   changing through time.
+	*/
         unsigned char *child_up;
-        int32_t       *fresh_children; //in the order of response
 
-        int32_t *child_errno;
+	/* @read_attempted:
+	   array of flags representing subvolumes where read operations of
+	   the read transaction have already been attempted. The array is
+	   first pre-filled with down subvolumes, and as reads are performed
+	   on other subvolumes, those are set as well. This way if the read
+	   operation fails we do not retry on that subvolume again.
+	*/
+	unsigned char *read_attempted;
 
-        dict_t  *xattr_req;
+	/* @readfn:
 
-        int32_t  inodelk_count;
-        int32_t  entrylk_count;
+	   pointer to function which will perform the read operation on a given
+	   subvolume. Used in read transactions.
+	*/
 
-        afr_internal_lock_t internal_lock;
+	afr_read_txn_wind_t readfn;
+
+	/* @refreshed:
+
+	   the inode was "refreshed" (i.e, pending xattrs from all subvols
+	   freshly inspected and inode ctx updated accordingly) as part of
+	   this transaction already.
+	*/
+	gf_boolean_t refreshed;
+
+	/* @inode:
+
+	   the inode on which the read txn is performed on. ref'ed and copied
+	   from either fd->inode or loc.inode
+	*/
+
+	inode_t *inode;
+
+	/* @parent[2]:
+
+	   parent inode[s] on which directory transactions are performed.
+	*/
+
+	inode_t *parent;
+	inode_t *parent2;
+
+	/* @readable:
+
+	   array of flags representing servers from which a read can be
+	   performed. This is the output of afr_inode_refresh()
+	*/
+	unsigned char *readable;
+
+	afr_inode_refresh_cbk_t refreshfn;
 
-        afr_locked_fd_t *locked_fd;
-        int32_t          source_child;
-        int32_t          lock_recovery_child;
+	/* @refreshinode:
+
+	   Inode currently getting refreshed.
+	*/
+	inode_t *refreshinode;
+
+	/*
+	  @pre_op_compat:
+
+	  compatibility mode of pre-op. send a separate pre-op and
+	  op operations as part of transaction, rather than combining
+	*/
+
+	gf_boolean_t pre_op_compat;
+
+        dict_t  *xattr_req;
+
+        afr_internal_lock_t internal_lock;
 
         dict_t  *dict;
+
         int      optimistic_change_log;
+	gf_boolean_t      delayed_post_op;
 
-        gf_boolean_t    fop_paused;
-        int (*fop_call_continue) (call_frame_t *frame, xlator_t *this);
+	/* Is the current writev() going to perform a stable write?
+	   i.e, is fd->flags or @flags writev param have O_SYNC or
+	   O_DSYNC?
+	*/
+	gf_boolean_t      stable_write;
+
+	/* This write appended to the file. Nnot necessarily O_APPEND,
+	   just means the offset of write was at the end of file.
+	*/
+	gf_boolean_t      append_write;
 
         /*
           This struct contains the arguments for the "continuation"
           (scheme-like) of fops
         */
 
-        int   op;
         struct {
                 struct {
                         unsigned char buf_set;
@@ -425,23 +422,6 @@ typedef struct _afr_local {
                 } statfs;
 
                 struct {
-                        uint32_t parent_entrylk;
-                        uuid_t  gfid_req;
-                        inode_t *inode;
-                        struct iatt buf;
-                        struct iatt postparent;
-                        dict_t **xattrs;
-                        dict_t *xattr;
-                        struct iatt *postparents;
-                        struct iatt *bufs;
-                        int32_t read_child;
-                        int32_t *sources;
-                        int32_t *success_children;
-                        gf_boolean_t fresh_lookup;
-                        gf_boolean_t possible_spb;
-                } lookup;
-
-                struct {
                         int32_t flags;
                 } open;
 
@@ -509,7 +489,9 @@ typedef struct _afr_local {
                 struct {
                         struct iatt prebuf;
                         struct iatt postbuf;
+                } inode_wfop; //common structure for all inode-write-fops
 
+                struct {
                         int32_t op_ret;
 
                         struct iovec *vector;
@@ -520,34 +502,21 @@ typedef struct _afr_local {
                 } writev;
 
                 struct {
-                        struct iatt prebuf;
-                        struct iatt postbuf;
-                } fsync;
-
-                struct {
                         off_t offset;
-                        struct iatt prebuf;
-                        struct iatt postbuf;
                 } truncate;
 
                 struct {
                         off_t offset;
-                        struct iatt prebuf;
-                        struct iatt postbuf;
                 } ftruncate;
 
                 struct {
                         struct iatt in_buf;
                         int32_t valid;
-                        struct iatt preop_buf;
-                        struct iatt postop_buf;
                 } setattr;
 
                 struct {
                         struct iatt in_buf;
                         int32_t valid;
-                        struct iatt preop_buf;
-                        struct iatt postop_buf;
                 } fsetattr;
 
                 struct {
@@ -575,90 +544,66 @@ typedef struct _afr_local {
                 /* dir write */
 
                 struct {
-                        fd_t *fd;
-                        dict_t *params;
-                        int32_t flags;
-                        mode_t mode;
                         inode_t *inode;
                         struct iatt buf;
                         struct iatt preparent;
                         struct iatt postparent;
-                        struct iatt read_child_buf;
+                        struct iatt prenewparent;
+                        struct iatt postnewparent;
+                } dir_fop; //common structure for all dir fops
+
+                struct {
+                        fd_t *fd;
+                        dict_t *params;
+                        int32_t flags;
+                        mode_t mode;
                 } create;
 
                 struct {
                         dev_t dev;
                         mode_t mode;
                         dict_t *params;
-                        inode_t *inode;
-                        struct iatt buf;
-                        struct iatt preparent;
-                        struct iatt postparent;
-                        struct iatt read_child_buf;
                 } mknod;
 
                 struct {
                         int32_t mode;
                         dict_t *params;
-                        inode_t *inode;
-                        struct iatt buf;
-                        struct iatt read_child_buf;
-                        struct iatt preparent;
-                        struct iatt postparent;
                 } mkdir;
 
                 struct {
-                        int32_t op_ret;
-                        int32_t op_errno;
-                        struct iatt preparent;
-                        struct iatt postparent;
-                } unlink;
-
-                struct {
-                        int   flags;
-                        int32_t op_ret;
-                        int32_t op_errno;
-                        struct iatt preparent;
-                        struct iatt postparent;
+                        int flags;
                 } rmdir;
 
                 struct {
-                        struct iatt buf;
-                        struct iatt read_child_buf;
-                        struct iatt preoldparent;
-                        struct iatt prenewparent;
-                        struct iatt postoldparent;
-                        struct iatt postnewparent;
-                } rename;
-
-                struct {
-                        inode_t *inode;
-                        struct iatt buf;
-                        struct iatt read_child_buf;
-                        struct iatt preparent;
-                        struct iatt postparent;
-                } link;
-
-                struct {
-                        inode_t *inode;
                         dict_t *params;
-                        struct iatt buf;
-                        struct iatt read_child_buf;
                         char *linkpath;
-                        struct iatt preparent;
-                        struct iatt postparent;
                 } symlink;
 
+		struct {
+			int32_t mode;
+			off_t offset;
+			size_t len;
+		} fallocate;
+
+		struct {
+			off_t offset;
+			size_t len;
+		} discard;
+
                 struct {
-                        int32_t flags;
-                        dir_entry_t *entries;
-                        int32_t count;
-                } setdents;
+                        off_t offset;
+                        off_t len;
+                        struct iatt prebuf;
+                        struct iatt postbuf;
+                } zerofill;
+
+
         } cont;
 
         struct {
                 off_t start, len;
 
+                gf_boolean_t    eager_lock_on;
                 int *eager_lock;
 
                 char *basename;
@@ -669,16 +614,67 @@ typedef struct _afr_local {
 
                 afr_transaction_type type;
 
-                int success_count;
-                int erase_pending;
-                int failure_count;
+		/* stub to resume on destruction
+		   of the transaction frame */
+		call_stub_t      *resume_stub;
+
+		struct list_head  eager_locked;
 
-                int last_tried;
-                int32_t *child_errno;
                 unsigned char   *pre_op;
 
+		/* @fop_subvols: subvolumes on which FOP will be attempted */
+                unsigned char   *fop_subvols;
+
+		/* @failed_subvols: subvolumes on which FOP failed. Always
+		   a subset of @fop_subvols */
+                unsigned char   *failed_subvols;
+
+		/* @dirtied: flag which indicates whether we set dirty flag
+		   in the OP. Typically true when we are performing operation
+		   on more than one subvol and optimistic changelog is disabled
+
+		   A 'true' value set in @dirtied flag means an 'undirtying'
+		   has to be done in POST-OP phase.
+		*/
+		gf_boolean_t  dirtied;
+
+		/* @inherited: flag which indicates that the dirty flags
+		   of the previous transaction were inherited
+		*/
+		gf_boolean_t  inherited;
+
+		/*
+		  @no_uninherit: flag which indicates that a pre_op_uninherit()
+		  must _not_ be attempted (and returned as failure) always. This
+		  flag is set when a hard pre-op is performed, but not accounted
+		  for it in fd_ctx->on_disk[]. Such transactions are "isolated"
+		  from the pre-op piggybacking entirely and therefore uninherit
+		  must not be attempted.
+		*/
+		gf_boolean_t no_uninherit;
+
+		/* @uninherit_done:
+		   @uninherit_value:
+
+		   The above pair variables make pre_op_uninherit() idempotent.
+		   Both are FALSE initially. The first call to pre_op_uninherit
+		   sets @uninherit_done to TRUE and the return value to
+		   @uninherit_value. Further calls will check for @uninherit_done
+		   to be TRUE and if so will simply return @uninherit_value.
+		*/
+		gf_boolean_t uninherit_done;
+		gf_boolean_t uninherit_value;
+
+		/* @changelog_resume: function to be called after changlogging
+		   (either pre-op or post-op) is done
+		*/
+
+		afr_changelog_resume_t changelog_resume;
+
                 call_frame_t *main_frame;
 
+                int (*wind) (call_frame_t *frame, xlator_t *this, int subvol);
+
                 int (*fop) (call_frame_t *frame, xlator_t *this);
 
                 int (*done) (call_frame_t *frame, xlator_t *this);
@@ -690,7 +686,7 @@ typedef struct _afr_local {
                 /* post-op hook */
         } transaction;
 
-        afr_self_heal_t self_heal;
+	syncbarrier_t barrier;
 
         struct marker_str     marker;
 
@@ -700,78 +696,75 @@ typedef struct _afr_local {
 
         mode_t          umask;
         int             xflag;
+        gf_boolean_t    do_discovery;
+	struct afr_reply *replies;
 } afr_local_t;
 
-typedef enum {
-        AFR_FD_NOT_OPENED,
-        AFR_FD_OPENED,
-        AFR_FD_OPENING
-} afr_fd_open_status_t;
-
-typedef struct {
-        struct list_head call_list;
-        call_frame_t    *frame;
-} afr_fd_paused_call_t;
-
-typedef struct {
-        unsigned int *pre_op_done;
-        afr_fd_open_status_t *opened_on; /* which subvolumes the fd is open on */
-        unsigned int *pre_op_piggyback;
-
-        unsigned int *lock_piggyback;
-        unsigned int *lock_acquired;
-
-        int flags;
-        uint64_t up_count;   /* number of CHILD_UPs this fd has seen */
-        uint64_t down_count; /* number of CHILD_DOWNs this fd has seen */
-
-        int32_t last_tried;
-
-        int  hit, miss;
-        gf_boolean_t failed_over;
-        struct list_head entries; /* needed for readdir failover */
-
-        unsigned char *locked_on; /* which subvolumes locks have been successful */
-	struct list_head  paused_calls; /* queued calls while fix_open happens  */
-} afr_fd_ctx_t;
-
-
-/* try alloc and if it fails, goto label */
-#define AFR_LOCAL_ALLOC_OR_GOTO(var, label) do {                    \
-                var = mem_get0 (THIS->local_pool);                  \
-                if (!var) {                                         \
-                        gf_log (this->name, GF_LOG_ERROR,           \
-                                "out of memory :(");                \
-                        op_errno = ENOMEM;                          \
-                        goto label;                                 \
-                }                                                   \
-        } while (0);
-
 
 /* did a call fail due to a child failing? */
 #define child_went_down(op_ret, op_errno) (((op_ret) < 0) &&            \
                                            ((op_errno == ENOTCONN) ||   \
                                             (op_errno == EBADFD)))
 
-#define afr_fop_failed(op_ret, op_errno) ((op_ret) == -1)
+int
+afr_inode_read_subvol_get (inode_t *inode, xlator_t *this,
+			   unsigned char *data_subvols,
+			   unsigned char *metadata_subvols,
+			   int *event_generation);
+int
+__afr_inode_read_subvol_get (inode_t *inode, xlator_t *this,
+			     unsigned char *data_subvols,
+			     unsigned char *metadata_subvols,
+			     int *event_generation);
 
-/* have we tried all children? */
-#define all_tried(i, count)  ((i) == (count) - 1)
+int
+__afr_inode_read_subvol_set (inode_t *inode, xlator_t *this,
+			     unsigned char *data_subvols,
+			     unsigned char *metadata_subvol,
+			     int event_generation);
+int
+afr_inode_read_subvol_set (inode_t *inode, xlator_t *this,
+			   unsigned char *data_subvols,
+			   unsigned char *metadata_subvols,
+			   int event_generation);
 
-int32_t
-afr_set_dict_gfid (dict_t *dict, uuid_t gfid);
+int
+afr_inode_read_subvol_reset (inode_t *inode, xlator_t *this);
+
+int
+afr_read_subvol_select_by_policy (inode_t *inode, xlator_t *this,
+				  unsigned char *readable);
 
 int
-pump_command_reply (call_frame_t *frame, xlator_t *this);
+afr_inode_read_subvol_type_get (inode_t *inode, xlator_t *this,
+				unsigned char *readable, int *event_p,
+				int type);
+int
+afr_read_subvol_get (inode_t *inode, xlator_t *this, int *subvol_p,
+		     int *event_p, afr_transaction_type type);
+
+#define afr_data_subvol_get(i, t, s, e) \
+	afr_read_subvol_get(i, t, s, e, AFR_DATA_TRANSACTION)
+
+#define afr_metadata_subvol_get(i, t, s, e) \
+	afr_read_subvol_get(i, t, s, e, AFR_METADATA_TRANSACTION)
+
+int
+afr_inode_refresh (call_frame_t *frame, xlator_t *this, inode_t *inode,
+		   afr_inode_refresh_cbk_t cbk);
 
 int32_t
 afr_notify (xlator_t *this, int32_t event, void *data, void *data2);
 
 int
-afr_attempt_lock_recovery (xlator_t *this, int32_t child_index);
+afr_init_entry_lockee (afr_entry_lockee_t *lockee, afr_local_t *local,
+                       loc_t *loc, char *basename, int child_count);
+
+void
+afr_entry_lockee_cleanup (afr_internal_lock_t *int_lock);
 
 int
-afr_save_locked_fd (xlator_t *this, fd_t *fd);
+afr_attempt_lock_recovery (xlator_t *this, int32_t child_index);
 
 int
 afr_mark_locked_nodes (xlator_t *this, fd_t *fd,
@@ -783,10 +776,6 @@ afr_set_lk_owner (call_frame_t *frame, xlator_t *this, void *lk_owner);
 int
 afr_set_lock_number (call_frame_t *frame, xlator_t *this);
 
-
-loc_t *
-lower_path (loc_t *l1, const char *b1, loc_t *l2, const char *b2);
-
 int32_t
 afr_unlock (call_frame_t *frame, xlator_t *this);
 
@@ -802,46 +791,30 @@ afr_blocking_lock (call_frame_t *frame, xlator_t *this);
 int
 afr_internal_lock_finish (call_frame_t *frame, xlator_t *this);
 
-void
-afr_lk_transfer_datalock (call_frame_t *dst, call_frame_t *src,
+int
+afr_lk_transfer_datalock (call_frame_t *dst, call_frame_t *src, char *dom,
                           unsigned int child_count);
 
-int pump_start (call_frame_t *frame, xlator_t *this);
-
 int
 __afr_fd_ctx_set (xlator_t *this, fd_t *fd);
 
 int
 afr_fd_ctx_set (xlator_t *this, fd_t *fd);
 
-int32_t
-afr_inode_get_read_ctx (xlator_t *this, inode_t *inode, int32_t *fresh_children);
-
-void
-afr_inode_set_read_ctx (xlator_t *this, inode_t *inode, int32_t read_child,
-                        int32_t *fresh_children);
+afr_fd_ctx_t *
+afr_fd_ctx_get (fd_t *fd, xlator_t *this);
 
 int
 afr_build_parent_loc (loc_t *parent, loc_t *child, int32_t *op_errno);
 
-unsigned int
-afr_up_children_count (unsigned char *child_up, unsigned int child_count);
-
-unsigned int
-afr_locked_children_count (unsigned char *children, unsigned int child_count);
-
-unsigned int
-afr_pre_op_done_children_count (unsigned char *pre_op,
-                                unsigned int child_count);
+int
+afr_locked_nodes_count (unsigned char *locked_nodes, int child_count);
 
-gf_boolean_t
-afr_is_fresh_lookup (loc_t *loc, xlator_t *this);
+int
+afr_replies_interpret (call_frame_t *frame, xlator_t *this, inode_t *inode);
 
 void
-afr_update_loc_gfids (loc_t *loc, struct iatt *buf, struct iatt *postparent);
-
-int
-afr_locked_nodes_count (unsigned char *locked_nodes, int child_count);
+afr_replies_wipe (afr_local_t *local, afr_private_t *priv);
 
 void
 afr_local_cleanup (afr_local_t *local, xlator_t *this);
@@ -849,31 +822,16 @@ afr_local_cleanup (afr_local_t *local, xlator_t *this);
 int
 afr_frame_return (call_frame_t *frame);
 
-gf_boolean_t
-afr_is_split_brain (xlator_t *this, inode_t *inode);
-
-void
-afr_set_split_brain (xlator_t *this, inode_t *inode, gf_boolean_t set);
-
 int
 afr_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
           fd_t *fd, dict_t *xdata);
 
 void
-afr_set_opendir_done (xlator_t *this, inode_t *inode);
-
-gf_boolean_t
-afr_is_opendir_done (xlator_t *this, inode_t *inode);
-
-void
 afr_local_transaction_cleanup (afr_local_t *local, xlator_t *this);
 
 int
 afr_cleanup_fd_ctx (xlator_t *this, fd_t *fd);
 
-int
-afr_launch_openfd_self_heal (call_frame_t *frame, xlator_t *this, fd_t *fd);
-
 #define AFR_STACK_UNWIND(fop, frame, params ...)                \
         do {                                                    \
                 afr_local_t *__local = NULL;                    \
@@ -904,7 +862,16 @@ afr_launch_openfd_self_heal (call_frame_t *frame, xlator_t *this, fd_t *fd);
                 }                                               \
         } while (0);
 
-#define AFR_NUM_CHANGE_LOGS            3 /*data + metadata + entry*/
+#define AFR_FRAME_INIT(frame, op_errno)				       \
+	({frame->local = mem_get0 (THIS->local_pool);		       \
+	if (afr_local_init (frame->local, THIS->private, &op_errno)) { \
+		afr_local_cleanup (frame->local, THIS);		       \
+		mem_put (frame->local);				       \
+		frame->local = NULL; };				       \
+	frame->local;})
+
+#define AFR_STACK_RESET(frame) do { int opr; STACK_RESET (frame->root); AFR_FRAME_INIT(frame, opr);} while (0)
+
 /* allocate and return a string that is the basename of argument */
 static inline char *
 AFR_BASENAME (const char *str)
@@ -917,6 +884,9 @@ AFR_BASENAME (const char *str)
         return __basename_str;
 }
 
+call_frame_t *
+afr_copy_frame (call_frame_t *base);
+
 int
 afr_transaction_local_init (afr_local_t *local, xlator_t *this);
 
@@ -924,9 +894,6 @@ int32_t
 afr_marker_getxattr (call_frame_t *frame, xlator_t *this,
                      loc_t *loc, const char *name,afr_local_t *local, afr_private_t *priv );
 
-int32_t *
-afr_children_create (int32_t child_count);
-
 int
 afr_local_init (afr_local_t *local, afr_private_t *priv, int32_t *op_errno);
 
@@ -935,99 +902,20 @@ afr_internal_lock_init (afr_internal_lock_t *lk, size_t child_count,
                         transaction_lk_type_t lk_type);
 
 int
-afr_first_up_child (unsigned char *child_up, size_t child_count);
+afr_higher_errno (int32_t old_errno, int32_t new_errno);
 
 int
-afr_select_read_child_from_policy (int32_t *fresh_children, int32_t child_count,
-                                   int32_t prev_read_child,
-                                   int32_t config_read_child, int32_t *sources);
-
-void
-afr_set_read_ctx_from_policy (xlator_t *this, inode_t *inode,
-                              int32_t *fresh_children, int32_t prev_read_child,
-                              int32_t config_read_child);
+afr_final_errno (afr_local_t *local, afr_private_t *priv);
 
-int32_t
-afr_get_call_child (xlator_t *this, unsigned char *child_up, int32_t read_child,
-                    int32_t *fresh_children,
-                    int32_t *call_child, int32_t *last_index);
-
-int32_t
-afr_next_call_child (int32_t *fresh_children, unsigned char *child_up,
-                     size_t child_count, int32_t *last_index,
-                     int32_t read_child);
-void
-afr_get_fresh_children (int32_t *success_children, int32_t *sources,
-                        int32_t *children, unsigned int child_count);
-void
-afr_children_add_child (int32_t *children, int32_t child,
-                              int32_t child_count);
-void
-afr_children_rm_child (int32_t *children, int32_t child,
-                             int32_t child_count);
-void
-afr_reset_children (int32_t *children, int32_t child_count);
-gf_boolean_t
-afr_error_more_important (int32_t old_errno, int32_t new_errno);
 int
-afr_errno_count (int32_t *children, int *child_errno,
-                 unsigned int child_count, int32_t op_errno);
-int
-afr_get_children_count (int32_t *children, unsigned int child_count);
-gf_boolean_t
-afr_is_child_present (int32_t *success_children, int32_t child_count,
-                      int32_t child);
-void
-afr_update_gfid_from_iatts (uuid_t uuid, struct iatt *bufs,
-                            int32_t *success_children,
-                            unsigned int child_count);
-void
-afr_reset_xattr (dict_t **xattr, unsigned int child_count);
-gf_boolean_t
-afr_conflicting_iattrs (struct iatt *bufs, int32_t *success_children,
-                        unsigned int child_count, const char *path,
-                        const char *xlator_name);
-unsigned int
-afr_gfid_missing_count (const char *xlator_name, int32_t *children,
-                        struct iatt *bufs, unsigned int child_count,
-                        const char *path);
-void
-afr_xattr_req_prepare (xlator_t *this, dict_t *xattr_req, const char *path);
-void
-afr_children_copy (int32_t *dst, int32_t *src, unsigned int child_count);
-afr_transaction_type
-afr_transaction_type_get (ia_type_t ia_type);
-int32_t
-afr_resultant_errno_get (int32_t *children,
-                         int *child_errno, unsigned int child_count);
-void
-afr_inode_rm_stale_children (xlator_t *this, inode_t *inode,
-                             int32_t *stale_children);
+afr_xattr_req_prepare (xlator_t *this, dict_t *xattr_req);
+
 void
-afr_launch_self_heal (call_frame_t *frame, xlator_t *this, inode_t *inode,
-                      gf_boolean_t background, ia_type_t ia_type, char *reason,
-                      void (*gfid_sh_success_cbk) (call_frame_t *sh_frame,
-                                                   xlator_t *this),
-                      int (*unwind) (call_frame_t *frame, xlator_t *this,
-                                     int32_t op_ret, int32_t op_errno,
-                                     int32_t sh_failed));
-int
-afr_fix_open (call_frame_t *frame, xlator_t *this, afr_fd_ctx_t *fd_ctx,
-              int need_open_count, int *need_open);
-int
-afr_open_fd_fix (call_frame_t *frame, xlator_t *this, gf_boolean_t pause_fop);
-int
-afr_set_elem_count_get (unsigned char *elems, int child_count);
+afr_fix_open (fd_t *fd, xlator_t *this);
 
 afr_fd_ctx_t *
 afr_fd_ctx_get (fd_t *fd, xlator_t *this);
 
-gf_boolean_t
-afr_open_only_data_self_heal (char *data_self_heal);
-
-gf_boolean_t
-afr_data_self_heal_enabled (char *data_self_heal);
-
 void
 afr_set_low_priority (call_frame_t *frame);
 int
@@ -1042,6 +930,10 @@ afr_matrix_cleanup (int32_t **pending, unsigned int m);
 
 int32_t**
 afr_matrix_create (unsigned int m, unsigned int n);
+
+void
+afr_filter_xattrs (dict_t *xattr);
+
 /*
  * Special value indicating we should use the "auto" quorum method instead of
  * a fixed value (including zero to turn off quorum enforcement).
@@ -1061,4 +953,24 @@ afr_matrix_create (unsigned int m, unsigned int n);
         }                                                                \
 } while (0);
 
+int
+afr_fd_report_unstable_write (xlator_t *this, fd_t *fd);
+
+gf_boolean_t
+afr_fd_has_witnessed_unstable_write (xlator_t *this, fd_t *fd);
+
+void
+afr_delayed_changelog_wake_resume (xlator_t *this, fd_t *fd, call_stub_t *stub);
+
+int
+afr_inodelk_init (afr_inodelk_t *lk, char *dom, size_t child_count);
+
+void
+afr_handle_open_fd_count (call_frame_t *frame, xlator_t *this);
+
+int
+afr_local_pathinfo (char *pathinfo, gf_boolean_t *is_local);
+
+void
+afr_remove_eager_lock_stub (afr_local_t *local);
 #endif /* __AFR_H__ */
diff --git a/xlators/cluster/afr/src/pump.c b/xlators/cluster/afr/src/pump.c
index eb3f84789..eed509956 100644
--- a/xlators/cluster/afr/src/pump.c
+++ b/xlators/cluster/afr/src/pump.c
@@ -21,6 +21,120 @@
 #include "afr-common.c"
 #include "defaults.c"
 #include "glusterfs.h"
+#include "pump.h"
+
+
+static int
+afr_set_dict_gfid (dict_t *dict, uuid_t gfid)
+{
+        int ret       = 0;
+        uuid_t *pgfid = NULL;
+
+        GF_ASSERT (gfid);
+
+        pgfid = GF_CALLOC (1, sizeof (uuid_t), gf_common_mt_char);
+        if (!pgfid) {
+                ret = -1;
+                goto out;
+        }
+
+        uuid_copy (*pgfid, gfid);
+
+        ret = dict_set_dynptr (dict, "gfid-req", pgfid, sizeof (uuid_t));
+        if (ret)
+                gf_log (THIS->name, GF_LOG_ERROR, "gfid set failed");
+
+out:
+        if (ret && pgfid)
+                GF_FREE (pgfid);
+	return ret;
+}
+
+static int
+afr_set_root_gfid (dict_t *dict)
+{
+        uuid_t gfid;
+        int ret = 0;
+
+        memset (gfid, 0, 16);
+        gfid[15] = 1;
+
+        ret = afr_set_dict_gfid (dict, gfid);
+
+        return ret;
+}
+
+static int
+afr_build_child_loc (xlator_t *this, loc_t *child, loc_t *parent, char *name)
+{
+        int   ret = -1;
+        uuid_t pargfid = {0};
+
+        if (!child)
+                goto out;
+
+        if (!uuid_is_null (parent->inode->gfid))
+                uuid_copy (pargfid, parent->inode->gfid);
+        else if (!uuid_is_null (parent->gfid))
+                uuid_copy (pargfid, parent->gfid);
+
+        if (uuid_is_null (pargfid))
+                goto out;
+
+        if (strcmp (parent->path, "/") == 0)
+                ret = gf_asprintf ((char **)&child->path, "/%s", name);
+        else
+                ret = gf_asprintf ((char **)&child->path, "%s/%s", parent->path,
+                                   name);
+
+        if (-1 == ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "asprintf failed while setting child path");
+        }
+
+        child->name = strrchr (child->path, '/');
+        if (child->name)
+                child->name++;
+
+        child->parent = inode_ref (parent->inode);
+        child->inode = inode_new (parent->inode->table);
+        uuid_copy (child->pargfid, pargfid);
+
+        if (!child->inode) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = 0;
+out:
+        if ((ret == -1) && child)
+                loc_wipe (child);
+
+        return ret;
+}
+
+static void
+afr_build_root_loc (xlator_t *this, loc_t *loc)
+{
+        afr_private_t   *priv = NULL;
+
+        priv = this->private;
+        loc->path = gf_strdup ("/");
+        loc->name = "";
+        loc->inode = inode_ref (priv->root_inode);
+        uuid_copy (loc->gfid, loc->inode->gfid);
+}
+
+static void
+afr_update_loc_gfids (loc_t *loc, struct iatt *buf, struct iatt *postparent)
+{
+        GF_ASSERT (loc);
+        GF_ASSERT (buf);
+
+        uuid_copy (loc->gfid, buf->ia_gfid);
+        if (postparent)
+                uuid_copy (loc->pargfid, postparent->ia_gfid);
+}
 
 static uint64_t pump_pid = 0;
 static inline void
@@ -387,54 +501,68 @@ gf_pump_traverse_directory (loc_t *loc)
                         if (ret)
                                 goto out;
 
-                        if (!IS_ENTRY_CWD (entry->d_name) &&
-                            !IS_ENTRY_PARENT (entry->d_name)) {
-
-                                    is_directory_empty = _gf_false;
-                                    gf_log (this->name, GF_LOG_DEBUG,
-                                            "lookup %s => %"PRId64,
-                                            entry_loc.path,
-                                            iatt.ia_ino);
-
-                                    ret = syncop_lookup (this, &entry_loc, NULL,
-                                                         &iatt, &xattr_rsp, &parent);
-
-                                    if (ret) {
-                                            gf_log (this->name, GF_LOG_ERROR,
-                                                    "%s: lookup failed",
-                                                    entry_loc.path);
-                                            continue;
-                                    }
-                                    pump_fill_loc_info (&entry_loc, &iatt,
-                                                       &parent);
-
-                                    pump_update_resume_state (this, entry_loc.path);
-
-                                    pump_save_path (this, entry_loc.path);
-                                    pump_save_file_stats (this, entry_loc.path);
-
-                                    ret = pump_check_and_update_status (this);
-                                    if (ret < 0) {
-                                            gf_log (this->name, GF_LOG_DEBUG,
-                                                    "Pump beginning to exit out");
-                                            goto out;
-                                    }
-
-                                    if (IA_ISDIR (iatt.ia_type)) {
-                                            if (is_pump_traversal_allowed (this, entry_loc.path)) {
-                                                    gf_log (this->name, GF_LOG_TRACE,
-                                                            "entering dir=%s",
-                                                            entry->d_name);
-                                                    gf_pump_traverse_directory (&entry_loc);
-                                            }
-                                    }
+			if ((strcmp (entry->d_name, ".") == 0) ||
+			    (strcmp (entry->d_name, "..") == 0))
+				continue;
+
+			is_directory_empty = _gf_false;
+			gf_log (this->name, GF_LOG_DEBUG,
+				"lookup %s => %"PRId64,
+				entry_loc.path,
+				iatt.ia_ino);
+
+			ret = syncop_lookup (this, &entry_loc, NULL, &iatt,
+					     &xattr_rsp, &parent);
+
+			if (ret) {
+				gf_log (this->name, GF_LOG_ERROR,
+					"%s: lookup failed", entry_loc.path);
+				continue;
+			}
+
+			ret = afr_selfheal_name (this, loc->gfid, entry->d_name);
+			if (ret) {
+				gf_log (this->name, GF_LOG_ERROR,
+					"%s: name self-heal failed (%s/%s)",
+					entry_loc.path, uuid_utoa (loc->gfid),
+					entry->d_name);
+				continue;
+			}
+
+			ret = afr_selfheal (this, iatt.ia_gfid);
+			if (ret) {
+				gf_log (this->name, GF_LOG_ERROR,
+					"%s: self-heal failed (%s)",
+					entry_loc.path, uuid_utoa (iatt.ia_gfid));
+				continue;
+			}
+
+			pump_fill_loc_info (&entry_loc, &iatt, &parent);
+
+			pump_update_resume_state (this, entry_loc.path);
+
+			pump_save_path (this, entry_loc.path);
+			pump_save_file_stats (this, entry_loc.path);
+
+			ret = pump_check_and_update_status (this);
+			if (ret < 0) {
+				gf_log (this->name, GF_LOG_DEBUG,
+					"Pump beginning to exit out");
+				goto out;
+			}
+
+			if (IA_ISDIR (iatt.ia_type)) {
+				if (is_pump_traversal_allowed (this, entry_loc.path)) {
+					gf_log (this->name, GF_LOG_TRACE,
+						"entering dir=%s", entry->d_name);
+					gf_pump_traverse_directory (&entry_loc);
+				}
                         }
                 }
 
                 gf_dirent_free (&entries);
                 free_entries = _gf_false;
-                gf_log (this->name, GF_LOG_TRACE,
-                        "offset incremented to %d",
+                gf_log (this->name, GF_LOG_TRACE, "offset incremented to %d",
                         (int32_t ) offset);
 
         }
@@ -443,7 +571,7 @@ gf_pump_traverse_directory (loc_t *loc)
         if (ret < 0)
                 gf_log (this->name, GF_LOG_DEBUG, "closing the fd failed");
 
-        if (is_directory_empty && IS_ROOT_PATH (loc->path)) {
+        if (is_directory_empty && (strcmp (loc->path, "/") == 0)) {
                pump_change_state (this, PUMP_STATE_RUNNING);
                gf_log (this->name, GF_LOG_INFO, "Empty source brick. "
                                 "Nothing to be done.");
@@ -496,17 +624,18 @@ pump_xattr_cleaner (call_frame_t *frame, void *cookie, xlator_t *this,
         afr_build_root_loc (this, &loc);
 
         ret = syncop_removexattr (priv->children[source], &loc,
-                                          PUMP_PATH);
+				  PUMP_PATH, 0);
 
         ret = syncop_removexattr (priv->children[sink], &loc,
-                                  PUMP_SINK_COMPLETE);
+                                  PUMP_SINK_COMPLETE, 0);
 
         for (i = 0; i < priv->child_count; i++) {
                 ret = syncop_removexattr (priv->children[i], &loc,
-                                          PUMP_SOURCE_COMPLETE);
-                if (ret)
+                                          PUMP_SOURCE_COMPLETE, 0);
+                if (ret) {
                         gf_log (this->name, GF_LOG_DEBUG, "removexattr "
-                                "failed with %s", strerror (errno));
+                                "failed with %s", strerror (-ret));
+                }
         }
 
         loc_wipe (&loc);
@@ -598,6 +727,7 @@ pump_lookup_sink (loc_t *loc)
         if (ret) {
                 gf_log (this->name, GF_LOG_DEBUG,
                         "Lookup on sink child failed");
+                ret = -1;
                 goto out;
         }
 
@@ -942,6 +1072,7 @@ pump_execute_status (call_frame_t *frame, xlator_t *this)
         uint64_t number_files = 0;
 
         char filename[PATH_MAX];
+        char summary[PATH_MAX+256];
         char *dict_str = NULL;
 
         int32_t op_ret = 0;
@@ -970,12 +1101,15 @@ pump_execute_status (call_frame_t *frame, xlator_t *this)
         }
 
         if (pump_priv->pump_finished) {
-        snprintf (dict_str, PATH_MAX + 256, "Number of files migrated = %"PRIu64"        Migration complete ",
-                  number_files);
+                snprintf (summary, PATH_MAX+256,
+                          "no_of_files=%"PRIu64, number_files);
         } else {
-        snprintf (dict_str, PATH_MAX + 256, "Number of files migrated = %"PRIu64"       Current file= %s ",
-                  number_files, filename);
+                snprintf (summary, PATH_MAX+256,
+                          "no_of_files=%"PRIu64":current_file=%s",
+                          number_files, filename);
         }
+        snprintf (dict_str, PATH_MAX+256, "status=%d:%s",
+                  (pump_priv->pump_finished)?1:0, summary);
 
         dict = dict_new ();
 
@@ -996,8 +1130,7 @@ out:
         if (dict)
                 dict_unref (dict);
 
-        if (dict_str)
-                GF_FREE (dict_str);
+        GF_FREE (dict_str);
 
         return 0;
 }
@@ -1272,127 +1405,16 @@ out:
 
 }
 
-struct _xattr_key {
-        char *key;
-        struct list_head list;
-};
-
-static void
-__gather_xattr_keys (dict_t *dict, char *key, data_t *value,
-                     void *data)
-{
-        struct list_head *  list  = data;
-        struct _xattr_key * xkey  = NULL;
-
-        if (!strncmp (key, AFR_XATTR_PREFIX,
-                      strlen (AFR_XATTR_PREFIX))) {
-
-                xkey = GF_CALLOC (1, sizeof (*xkey), gf_afr_mt_xattr_key);
-                if (!xkey)
-                        return;
-
-                xkey->key = key;
-                INIT_LIST_HEAD (&xkey->list);
-
-                list_add_tail (&xkey->list, list);
-        }
-}
-
-static void
-__filter_xattrs (dict_t *dict)
-{
-        struct list_head keys;
-
-        struct _xattr_key *key;
-        struct _xattr_key *tmp;
-
-        INIT_LIST_HEAD (&keys);
-
-        dict_foreach (dict, __gather_xattr_keys,
-                      (void *) &keys);
-
-        list_for_each_entry_safe (key, tmp, &keys, list) {
-                dict_del (dict, key->key);
-
-                list_del_init (&key->list);
-
-                GF_FREE (key);
-        }
-}
-
-int32_t
-pump_getxattr_cbk (call_frame_t *frame, void *cookie,
-		  xlator_t *this, int32_t op_ret, int32_t op_errno,
-		  dict_t *dict, dict_t *xdata)
-{
-	afr_private_t   *priv           = NULL;
-	afr_local_t     *local          = NULL;
-	xlator_t        **children      = NULL;
-	int             unwind          = 1;
-        int32_t         *last_index     = NULL;
-        int32_t         next_call_child = -1;
-        int32_t         read_child      = -1;
-        int32_t         *fresh_children = NULL;
-
-
-	priv     = this->private;
-	children = priv->children;
-
-	local = frame->local;
-
-        read_child = (long) cookie;
-
-	if (op_ret == -1) {
-		last_index = &local->cont.getxattr.last_index;
-                fresh_children = local->fresh_children;
-                next_call_child = afr_next_call_child (fresh_children,
-                                                       local->child_up,
-                                                       priv->child_count,
-                                                       last_index, read_child);
-                if (next_call_child < 0)
-                        goto out;
-
-		unwind = 0;
-		STACK_WIND_COOKIE (frame, pump_getxattr_cbk,
-				   (void *) (long) read_child,
-				   children[next_call_child],
-				   children[next_call_child]->fops->getxattr,
-				   &local->loc,
-				   local->cont.getxattr.name, NULL);
-	}
-
-out:
-	if (unwind) {
-                if (op_ret >= 0 && dict)
-                        __filter_xattrs (dict);
-
-		AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, NULL);
-	}
-
-	return 0;
-}
-
-int32_t
-pump_getxattr (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, const char *name, dict_t *xdata)
+int
+pump_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+	       const char *name, dict_t *xdata)
 {
-	afr_private_t *   priv       = NULL;
-	xlator_t **       children   = NULL;
-	int               call_child = 0;
-	afr_local_t       *local     = NULL;
-	int32_t           ret     = -1;
-	int32_t           op_errno   = 0;
-        uint64_t          read_child = 0;
-
-
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
+	afr_private_t *priv = NULL;
+	int op_errno = 0;
+	int ret = 0;
 
-	priv     = this->private;
-	VALIDATE_OR_GOTO (priv->children, out);
+	priv = this->private;
 
-	children = priv->children;
         if (!priv->use_afr_in_pump) {
                 STACK_WIND (frame, default_getxattr_cbk,
                             FIRST_CHILD (this),
@@ -1401,14 +1423,6 @@ pump_getxattr (call_frame_t *frame, xlator_t *this,
                 return 0;
         }
 
-
-	AFR_LOCAL_ALLOC_OR_GOTO (frame->local, out);
-	local = frame->local;
-
-        ret = afr_local_init (local, priv, &op_errno);
-        if (ret < 0)
-                goto out;
-
         if (name) {
                 if (!strncmp (name, AFR_XATTR_PREFIX,
                               strlen (AFR_XATTR_PREFIX))) {
@@ -1426,32 +1440,7 @@ pump_getxattr (call_frame_t *frame, xlator_t *this,
                 }
         }
 
-        local->fresh_children = GF_CALLOC (priv->child_count,
-                                          sizeof (*local->fresh_children),
-                                          gf_afr_mt_int32_t);
-        if (!local->fresh_children) {
-                ret = -1;
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        read_child = afr_inode_get_read_ctx (this, loc->inode, local->fresh_children);
-        ret = afr_get_call_child (this, local->child_up, read_child,
-                                     local->fresh_children,
-                                     &call_child,
-                                     &local->cont.getxattr.last_index);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
-        }
-	loc_copy (&local->loc, loc);
-	if (name)
-	  local->cont.getxattr.name       = gf_strdup (name);
-
-	STACK_WIND_COOKIE (frame, pump_getxattr_cbk,
-			   (void *) (long) call_child,
-			   children[call_child], children[call_child]->fops->getxattr,
-			   loc, name, xdata);
+	afr_getxattr (frame, this, loc, name, xdata);
 
 	ret = 0;
 out:
@@ -1460,134 +1449,6 @@ out:
 	return 0;
 }
 
-static int
-afr_setxattr_unwind (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t *   local = NULL;
-	call_frame_t   *main_frame = NULL;
-
-	local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		if (local->transaction.main_frame)
-			main_frame = local->transaction.main_frame;
-		local->transaction.main_frame = NULL;
-	}
-	UNLOCK (&frame->lock);
-
-	if (main_frame) {
-		AFR_STACK_UNWIND (setxattr, main_frame,
-                                  local->op_ret, local->op_errno, NULL);
-	}
-	return 0;
-}
-
-static int
-afr_setxattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		       int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
-
-	int call_count  = -1;
-	int need_unwind = 0;
-
-	local = frame->local;
-	priv = this->private;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret != -1) {
-			if (local->success_count == 0) {
-				local->op_ret = op_ret;
-			}
-			local->success_count++;
-
-			if (local->success_count == priv->child_count) {
-				need_unwind = 1;
-			}
-		}
-
-		local->op_errno = op_errno;
-	}
-	UNLOCK (&frame->lock);
-
-	if (need_unwind)
-		local->transaction.unwind (frame, this);
-
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0) {
-		local->transaction.resume (frame, this);
-	}
-
-	return 0;
-}
-
-static int
-afr_setxattr_wind (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-
-	int call_count = -1;
-	int i = 0;
-
-	local = frame->local;
-	priv = this->private;
-
-	call_count = afr_up_children_count (local->child_up, priv->child_count);
-
-	if (call_count == 0) {
-		local->transaction.resume (frame, this);
-		return 0;
-	}
-
-	local->call_count = call_count;
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (local->child_up[i]) {
-			STACK_WIND_COOKIE (frame, afr_setxattr_wind_cbk,
-					   (void *) (long) i,
-					   priv->children[i],
-					   priv->children[i]->fops->setxattr,
-					   &local->loc,
-					   local->cont.setxattr.dict,
-					   local->cont.setxattr.flags, NULL);
-
-			if (!--call_count)
-				break;
-		}
-	}
-
-	return 0;
-}
-
-
-static int
-afr_setxattr_done (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t * local = frame->local;
-
-	local->transaction.unwind (frame, this);
-
-	AFR_STACK_DESTROY (frame);
-
-	return 0;
-}
-
-int32_t
-pump_setxattr_cbk (call_frame_t *frame,
-		      void *cookie,
-		      xlator_t *this,
-		      int32_t op_ret,
-		      int32_t op_errno, dict_t *xdata)
-{
-	AFR_STACK_UNWIND (setxattr, frame, op_ret, op_errno, xdata);
-	return 0;
-}
-
 int
 pump_command_reply (call_frame_t *frame, xlator_t *this)
 {
@@ -1611,52 +1472,56 @@ pump_command_reply (call_frame_t *frame, xlator_t *this)
 }
 
 int
-pump_parse_command (call_frame_t *frame, xlator_t *this,
-                    afr_local_t *local, dict_t *dict)
+pump_parse_command (call_frame_t *frame, xlator_t *this, dict_t *dict,
+		    int *op_errno_p)
 {
-
+	afr_local_t *local = NULL;
         int ret = -1;
+	int op_errno = 0;
 
         if (pump_command_start (this, dict)) {
-                frame->local = local;
+                local = AFR_FRAME_INIT (frame, op_errno);
+		if (!local)
+			goto out;
                 local->dict = dict_ref (dict);
                 ret = pump_execute_start (frame, this);
 
         } else if (pump_command_pause (this, dict)) {
-                frame->local = local;
+		local = AFR_FRAME_INIT (frame, op_errno);
+		if (!local)
+			goto out;
                 local->dict = dict_ref (dict);
                 ret = pump_execute_pause (frame, this);
 
         } else if (pump_command_abort (this, dict)) {
-                frame->local = local;
+		local = AFR_FRAME_INIT (frame, op_errno);
+		if (!local)
+			goto out;
                 local->dict = dict_ref (dict);
                 ret = pump_execute_abort (frame, this);
 
         } else if (pump_command_commit (this, dict)) {
-                frame->local = local;
+		local = AFR_FRAME_INIT (frame, op_errno);
+		if (!local)
+			goto out;
                 local->dict = dict_ref (dict);
                 ret = pump_execute_commit (frame, this);
         }
+out:
+	if (op_errno_p)
+		*op_errno_p = op_errno;
         return ret;
 }
 
 int
-pump_setxattr (call_frame_t *frame, xlator_t *this,
-               loc_t *loc, dict_t *dict, int32_t flags, dict_t *xdata)
+pump_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+	       int32_t flags, dict_t *xdata)
 {
-	afr_private_t * priv  = NULL;
-	afr_local_t   * local = NULL;
-	call_frame_t   *transaction_frame = NULL;
-        data_pair_t   * trav  = NULL;
+	afr_private_t *priv = NULL;
 	int ret = -1;
 	int op_errno = 0;
 
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
-
-        GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.pump*", dict,
-                                   trav, op_errno, out);
+        GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.pump*", dict, op_errno, out);
 
 	priv = this->private;
         if (!priv->use_afr_in_pump) {
@@ -1667,56 +1532,15 @@ pump_setxattr (call_frame_t *frame, xlator_t *this,
                 return 0;
         }
 
-
-	AFR_LOCAL_ALLOC_OR_GOTO (local, out);
-
-	ret = afr_local_init (local, priv, &op_errno);
-	if (ret < 0) {
-                afr_local_cleanup (local, this);
-		goto out;
-        }
-
-        ret = pump_parse_command (frame, this,
-                                  local, dict);
-        if (ret >= 0) {
-                ret = 0;
+        ret = pump_parse_command (frame, this, dict, &op_errno);
+        if (ret >= 0)
                 goto out;
-        }
 
-	transaction_frame = copy_frame (frame);
-	if (!transaction_frame) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory.");
-                op_errno = ENOMEM;
-                ret = -1;
-                afr_local_cleanup (local, this);
-		goto out;
-	}
-
-	transaction_frame->local = local;
-
-	local->op_ret = -1;
-
-	local->cont.setxattr.dict  = dict_ref (dict);
-	local->cont.setxattr.flags = flags;
-
-	local->transaction.fop    = afr_setxattr_wind;
-	local->transaction.done   = afr_setxattr_done;
-	local->transaction.unwind = afr_setxattr_unwind;
-
-	loc_copy (&local->loc, loc);
-
-	local->transaction.main_frame = frame;
-	local->transaction.start   = LLONG_MAX - 1;
-	local->transaction.len     = 0;
-
-	afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+	afr_setxattr (frame, this, loc, dict, flags, xdata);
 
 	ret = 0;
 out:
 	if (ret < 0) {
-		if (transaction_frame)
-			AFR_STACK_DESTROY (transaction_frame);
 		AFR_STACK_UNWIND (setxattr, frame, -1, op_errno, NULL);
 	}
 
@@ -2384,7 +2208,7 @@ int32_t
 init (xlator_t *this)
 {
 	afr_private_t * priv        = NULL;
-        pump_private_t *pump_priv = NULL;
+        pump_private_t *pump_priv   = NULL;
 	int             child_count = 0;
 	xlator_list_t * trav        = NULL;
 	int             i           = 0;
@@ -2405,17 +2229,11 @@ init (xlator_t *this)
 			"Volume is dangling.");
 	}
 
-	this->private = GF_CALLOC (1, sizeof (afr_private_t),
-                                   gf_afr_mt_afr_private_t);
-        if (!this->private)
+	priv = GF_CALLOC (1, sizeof (afr_private_t), gf_afr_mt_afr_private_t);
+        if (!priv)
                 goto out;
 
-	priv = this->private;
         LOCK_INIT (&priv->lock);
-        LOCK_INIT (&priv->read_child_lock);
-        //lock recovery is not done in afr
-        pthread_mutex_init (&priv->mutex, NULL);
-        INIT_LIST_HEAD (&priv->saved_fds);
 
         child_count = xlator_subvolume_count (this);
         if (child_count != 2) {
@@ -2434,14 +2252,13 @@ init (xlator_t *this)
 	priv->metadata_self_heal = 1;
 	priv->entry_self_heal    = 1;
 
-        priv->data_self_heal_algorithm = "";
-
         priv->data_self_heal_window_size = 16;
 
 	priv->data_change_log     = 1;
 	priv->metadata_change_log = 1;
 	priv->entry_change_log    = 1;
         priv->use_afr_in_pump = 1;
+        priv->sh_readdir_size = 65536;
 
 	/* Locking options */
 
@@ -2450,9 +2267,6 @@ init (xlator_t *this)
            and the sink.
         */
 
-	priv->strict_readdir = _gf_false;
-
-	priv->wait_count = 1;
 	priv->child_up = GF_CALLOC (sizeof (unsigned char), child_count,
                                  gf_afr_mt_char);
 	if (!priv->child_up) {
@@ -2486,7 +2300,8 @@ init (xlator_t *this)
 	while (i < child_count) {
 		priv->children[i] = trav->xlator;
 
-                ret = gf_asprintf (&priv->pending_key[i], "%s.%s", AFR_XATTR_PREFIX,
+                ret = gf_asprintf (&priv->pending_key[i], "%s.%s",
+                                   AFR_XATTR_PREFIX,
                                    trav->xlator->name);
                 if (-1 == ret) {
                         gf_log (this->name, GF_LOG_ERROR,
@@ -2499,7 +2314,12 @@ init (xlator_t *this)
 		i++;
 	}
 
-        priv->first_lookup = 1;
+        ret = gf_asprintf (&priv->sh_domain, "%s-self-heal", this->name);
+        if (-1 == ret) {
+                op_errno = ENOMEM;
+                goto out;
+        }
+
         priv->root_inode = NULL;
 
         priv->last_event = GF_CALLOC (child_count, sizeof (*priv->last_event),
@@ -2524,13 +2344,12 @@ init (xlator_t *this)
         pump_priv->resume_path = GF_CALLOC (1, PATH_MAX,
                                             gf_afr_mt_char);
         if (!pump_priv->resume_path) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory");
+                gf_log (this->name, GF_LOG_ERROR, "Out of memory");
                 ret = -1;
                 goto out;
         }
 
-	pump_priv->env = syncenv_new (0);
+	pump_priv->env = this->ctx->env;
         if (!pump_priv->env) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "Could not create new sync-environment");
@@ -2548,11 +2367,32 @@ init (xlator_t *this)
         }
 
 	priv->pump_private = pump_priv;
+        pump_priv = NULL;
+
+        this->private = priv;
+        priv = NULL;
 
         pump_change_state (this, PUMP_STATE_ABORT);
 
 	ret = 0;
 out:
+
+        if (pump_priv) {
+                GF_FREE (pump_priv->resume_path);
+                LOCK_DESTROY (&pump_priv->resume_path_lock);
+                LOCK_DESTROY (&pump_priv->pump_state_lock);
+                GF_FREE (pump_priv);
+        }
+
+        if (priv) {
+                GF_FREE (priv->child_up);
+                GF_FREE (priv->children);
+                GF_FREE (priv->pending_key);
+                GF_FREE (priv->last_event);
+                LOCK_DESTROY (&priv->lock);
+                GF_FREE (priv);
+        }
+
 	return ret;
 }
 
@@ -2571,9 +2411,6 @@ fini (xlator_t *this)
         if (!pump_priv)
                 goto afr_priv;
 
-        if (pump_priv->env)
-                syncenv_destroy (pump_priv->env);
-
         GF_FREE (pump_priv->resume_path);
         LOCK_DESTROY (&pump_priv->resume_path_lock);
         LOCK_DESTROY (&pump_priv->pump_state_lock);
diff --git a/xlators/cluster/afr/src/pump.h b/xlators/cluster/afr/src/pump.h
index bc4c31a78..9d0b6db6a 100644
--- a/xlators/cluster/afr/src/pump.h
+++ b/xlators/cluster/afr/src/pump.h
@@ -75,4 +75,7 @@ pump_command_status (xlator_t *this, dict_t *dict);
 int
 pump_execute_status (call_frame_t *frame, xlator_t *this);
 
+int
+pump_command_reply (call_frame_t *frame, xlator_t *this);
+
 #endif /* __PUMP_H__ */
diff --git a/xlators/cluster/dht/src/Makefile.am b/xlators/cluster/dht/src/Makefile.am
index e35058d65..174bea841 100644
--- a/xlators/cluster/dht/src/Makefile.am
+++ b/xlators/cluster/dht/src/Makefile.am
@@ -4,7 +4,7 @@ xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/cluster
 
 dht_common_source = dht-layout.c dht-helper.c dht-linkfile.c dht-rebalance.c \
 	dht-selfheal.c dht-rename.c dht-hashfn.c dht-diskusage.c \
-	dht-common.c dht-inode-write.c dht-inode-read.c \
+	dht-common.c dht-inode-write.c dht-inode-read.c dht-shared.c \
 	$(top_builddir)/xlators/lib/src/libxlator.c
 
 dht_la_SOURCES = $(dht_common_source) dht.c
@@ -12,22 +12,23 @@ dht_la_SOURCES = $(dht_common_source) dht.c
 nufa_la_SOURCES = $(dht_common_source) nufa.c
 switch_la_SOURCES = $(dht_common_source) switch.c
 
-dht_la_LDFLAGS = -module -avoidversion
+dht_la_LDFLAGS = -module -avoid-version
 dht_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-nufa_la_LDFLAGS = -module -avoidversion
+nufa_la_LDFLAGS = -module -avoid-version
 nufa_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-switch_la_LDFLAGS = -module -avoidversion
+switch_la_LDFLAGS = -module -avoid-version
 switch_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = dht-common.h dht-mem-types.h \
 	$(top_builddir)/xlators/lib/src/libxlator.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS) \
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
 	-I$(top_srcdir)/xlators/lib/src
 
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
 CLEANFILES =
 
 uninstall-local:
diff --git a/xlators/cluster/dht/src/dht-common.c b/xlators/cluster/dht/src/dht-common.c
index aedc453e8..3868fc38f 100644
--- a/xlators/cluster/dht/src/dht-common.c
+++ b/xlators/cluster/dht/src/dht-common.c
@@ -22,17 +22,18 @@
 #include "dht-common.h"
 #include "defaults.h"
 #include "byte-order.h"
+#include "glusterfs-acl.h"
 
 #include <sys/time.h>
 #include <libgen.h>
 
-void
+int
 dht_aggregate (dict_t *this, char *key, data_t *value, void *data)
 {
         dict_t  *dst  = NULL;
         int64_t *ptr  = 0, *size = NULL;
         int32_t  ret  = -1;
-        data_pair_t  *data_pair = NULL;
+        data_t  *dict_data = NULL;
 
         dst = data;
 
@@ -44,30 +45,35 @@ dht_aggregate (dict_t *this, char *key, data_t *value, void *data)
                         if (size == NULL) {
                                 gf_log ("dht", GF_LOG_WARNING,
                                         "memory allocation failed");
-                                return;
+                                return -1;
                         }
                         ret = dict_set_bin (dst, key, size, sizeof (int64_t));
                         if (ret < 0) {
                                 gf_log ("dht", GF_LOG_WARNING,
                                         "dht aggregate dict set failed");
                                 GF_FREE (size);
-                                return;
+                                return -1;
                         }
                 }
 
                 ptr = data_to_bin (value);
                 if (ptr == NULL) {
                         gf_log ("dht", GF_LOG_WARNING, "data to bin failed");
-                        return;
+                        return -1;
                 }
 
                 *size = hton64 (ntoh64 (*size) + ntoh64 (*ptr));
+
+        } else if (fnmatch (GF_XATTR_STIME_PATTERN, key, FNM_NOESCAPE) == 0) {
+                ret = gf_get_min_stime (THIS, dst, key, value);
+                if (ret < 0)
+                        return ret;
         } else {
                 /* compare user xattrs only */
                 if (!strncmp (key, "user.", strlen ("user."))) {
-                        ret = dict_lookup (dst, key, &data_pair); 
-                        if (!ret && data_pair && value) {
-                                ret = is_data_equal (data_pair->value, value);
+                        ret = dict_lookup (dst, key, &dict_data);
+                        if (!ret && dict_data && value) {
+                                ret = is_data_equal (dict_data, value);
                                 if (!ret)
                                         gf_log ("dht", GF_LOG_DEBUG,
                                                 "xattr mismatch for %s", key);
@@ -78,7 +84,7 @@ dht_aggregate (dict_t *this, char *key, data_t *value, void *data)
                         gf_log ("dht", GF_LOG_WARNING, "xattr dict set failed");
         }
 
-        return;
+        return 0;
 }
 
 
@@ -125,7 +131,10 @@ dht_lookup_selfheal_cbk (call_frame_t *frame, void *cookie,
                 ret = dht_layout_set (this, local->inode, layout);
         }
 
-        WIPE (&local->postparent);
+        if (local->loc.parent) {
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           &local->postparent, 1);
+        }
 
         DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
 
@@ -145,9 +154,11 @@ dht_discover_complete (xlator_t *this, call_frame_t *discover_frame)
         int              op_errno = 0;
         int              ret = -1;
         dht_layout_t    *layout = NULL;
+        dht_conf_t      *conf = NULL;
 
         local = discover_frame->local;
         layout = local->layout;
+        conf = this->private;
 
         LOCK(&discover_frame->lock);
         {
@@ -190,11 +201,14 @@ dht_discover_complete (xlator_t *this, call_frame_t *discover_frame)
                                 "(overlaps/holes present: %s, "
                                 "ENOENT errors: %d)", local->loc.path,
                                 (ret < 0) ? "yes" : "no", (ret > 0) ? ret : 0);
-                        op_errno = EINVAL;
-                        goto out;
+                        if ((ret > 0) && (ret == conf->subvolume_cnt)) {
+                                op_errno = ESTALE;
+                                goto out;
+                        }
                 }
 
-                dht_layout_set (this, local->inode, layout);
+                if (local->inode)
+                        dht_layout_set (this, local->inode, layout);
         }
 
         DHT_STACK_UNWIND (lookup, main_frame, local->op_ret, local->op_errno,
@@ -223,6 +237,7 @@ dht_discover_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         int           is_dir                  = 0;
         int           is_linkfile             = 0;
         int           attempt_unwind          = 0;
+        dht_conf_t   *conf                    = 0;
 
         GF_VALIDATE_OR_GOTO ("dht", frame, out);
         GF_VALIDATE_OR_GOTO ("dht", this, out);
@@ -232,6 +247,7 @@ dht_discover_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         local = frame->local;
         prev  = cookie;
+        conf = this->private;
 
         layout = local->layout;
 
@@ -266,7 +282,8 @@ dht_discover_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         goto unlock;
                 }
 
-                is_linkfile = check_is_linkfile (inode, stbuf, xattr);
+                is_linkfile = check_is_linkfile (inode, stbuf, xattr,
+                                                 conf->link_xattr_name);
                 is_dir = check_is_dir (inode, stbuf, xattr);
 
                 if (is_dir) {
@@ -325,23 +342,20 @@ dht_discover (call_frame_t *frame, xlator_t *this, loc_t *loc)
         int          i = 0;
         call_frame_t *discover_frame = NULL;
 
-
         conf = this->private;
         local = frame->local;
 
-        ret = dict_set_uint32 (local->xattr_req,
-                               "trusted.glusterfs.dht", 4 * 4);
+        ret = dict_set_uint32 (local->xattr_req, conf->xattr_name, 4 * 4);
         if (ret)
                 gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to set 'trusted.glusterfs.dht' key",
-                        loc->path);
+                        "%s: failed to set '%s' key",
+                        loc->path, conf->xattr_name);
 
-        ret = dict_set_uint32 (local->xattr_req,
-                               "trusted.glusterfs.dht.linkto", 256);
+        ret = dict_set_uint32 (local->xattr_req, conf->link_xattr_name, 256);
         if (ret)
                 gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to set 'trusted.glusterfs.dht.linkto' key",
-                        loc->path);
+                        "%s: failed to set '%s' key",
+                        loc->path, conf->link_xattr_name);
 
         call_cnt        = conf->subvolume_cnt;
         local->call_cnt = call_cnt;
@@ -427,7 +441,7 @@ dht_lookup_dir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                         op_ret, op_errno, xattr);
 
                 if (op_ret == -1) {
-                        local->op_errno = ENOENT;
+                        local->op_errno = op_errno;
                         gf_log (this->name, GF_LOG_DEBUG,
                                 "lookup of %s on %s returned error (%s)",
                                 local->loc.path, prev->this->name,
@@ -486,6 +500,11 @@ unlock:
                         dht_layout_set (this, local->inode, layout);
                 }
 
+                if (local->loc.parent) {
+                        dht_inode_ctx_time_update (local->loc.parent, this,
+                                                   &local->postparent, 1);
+                }
+
                 DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
                 DHT_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
                                   local->inode, &local->stbuf, local->xattr,
@@ -517,6 +536,8 @@ dht_revalidate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         int           ret  = -1;
         int           is_dir = 0;
         int           is_linkfile = 0;
+        call_frame_t *copy          = NULL;
+        dht_local_t  *copy_local    = NULL;
 
         GF_VALIDATE_OR_GOTO ("dht", frame, err);
         GF_VALIDATE_OR_GOTO ("dht", this, err);
@@ -575,7 +596,8 @@ dht_revalidate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 layout = local->layout;
 
                 is_dir = check_is_dir (inode, stbuf, xattr);
-                is_linkfile = check_is_linkfile (inode, stbuf, xattr);
+                is_linkfile = check_is_linkfile (inode, stbuf, xattr,
+                                                 conf->link_xattr_name);
 
                 if (is_linkfile) {
                         gf_log (this->name, GF_LOG_INFO,
@@ -587,6 +609,23 @@ dht_revalidate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 }
 
                 if (is_dir) {
+                        ret = dht_dir_has_layout (xattr, conf->xattr_name);
+                        if (ret >= 0) {
+                                if (is_greater_time(local->stbuf.ia_ctime,
+                                                    local->stbuf.ia_ctime_nsec,
+                                                    stbuf->ia_ctime,
+                                                    stbuf->ia_ctime_nsec)) {
+                                        local->prebuf.ia_gid = stbuf->ia_gid;
+                                        local->prebuf.ia_uid = stbuf->ia_uid;
+                                }
+                        }
+                        if (local->stbuf.ia_type != IA_INVAL)
+                        {
+                                if ((local->stbuf.ia_gid != stbuf->ia_gid) ||
+                                    (local->stbuf.ia_uid != stbuf->ia_uid)) {
+                                        local->need_selfheal = 1;
+                                }
+                        }
                         ret = dht_layout_dir_mismatch (this, layout,
                                                        prev->this, &local->loc,
                                                        xattr);
@@ -625,7 +664,28 @@ out:
                     && (conf && conf->unhashed_sticky_bit)) {
                         local->stbuf.ia_prot.sticky = 1;
                 }
-                if (local->layout_mismatch) {
+                if (local->need_selfheal) {
+                        local->need_selfheal = 0;
+                        uuid_copy (local->gfid, local->stbuf.ia_gfid);
+                        local->stbuf.ia_gid = local->prebuf.ia_gid;
+                        local->stbuf.ia_uid = local->prebuf.ia_uid;
+                        copy = create_frame (this, this->ctx->pool);
+                        if (copy) {
+                                copy_local = dht_local_init (copy, &local->loc,
+                                                             NULL, 0);
+                                if (!copy_local)
+                                        goto cont;
+                                copy_local->stbuf = local->stbuf;
+                                copy->local = copy_local;
+                                FRAME_SU_DO (copy, dht_local_t);
+                                ret = synctask_new (this->ctx->env,
+                                                    dht_dir_attr_heal,
+                                                    dht_dir_attr_heal_done,
+                                                    copy, copy);
+                        }
+                }
+cont:
+		if (local->layout_mismatch) {
                         /* Found layout mismatch in the directory, need to
                            fix this in the inode context */
                         dht_layout_unref (this, local->layout);
@@ -651,7 +711,10 @@ out:
                         local->op_errno = ESTALE;
                 }
 
-                WIPE (&local->postparent);
+                if (local->loc.parent) {
+                        dht_inode_ctx_time_update (local->loc.parent, this,
+                                                   &local->postparent, 1);
+                }
 
                 DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
                 DHT_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
@@ -687,7 +750,7 @@ dht_lookup_linkfile_create_cbk (call_frame_t *frame, void *cookie,
         cached_subvol = local->cached_subvol;
         conf = this->private;
 
-        ret = dht_layout_preset (this, local->cached_subvol, inode);
+        ret = dht_layout_preset (this, local->cached_subvol, local->loc.inode);
         if (ret < 0) {
                 gf_log (this->name, GF_LOG_DEBUG,
                         "failed to set layout for subvolume %s",
@@ -703,8 +766,12 @@ dht_lookup_linkfile_create_cbk (call_frame_t *frame, void *cookie,
                 local->stbuf.ia_prot.sticky = 1;
         }
 
+        if (local->loc.parent) {
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           postparent, 1);
+        }
+
 unwind:
-        WIPE (&local->postparent);
         if (local->linked == _gf_true)
                 dht_linkfile_attr_heal (frame, this);
 
@@ -779,7 +846,10 @@ dht_lookup_everywhere_done (call_frame_t *frame, xlator_t *this)
                                                   "<nil>"));
                 }
 
-                WIPE (&local->postparent);
+                if (local->loc.parent) {
+                        dht_inode_ctx_time_update (local->loc.parent, this,
+                                                   &local->postparent, 1);
+                }
 
                 DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
                 DHT_STACK_UNWIND (lookup, frame, local->op_ret,
@@ -809,7 +879,10 @@ dht_lookup_everywhere_done (call_frame_t *frame, xlator_t *this)
                         local->op_errno = EINVAL;
                 }
 
-                WIPE (&local->postparent);
+                if (local->loc.parent) {
+                        dht_inode_ctx_time_update (local->loc.parent, this,
+                                                   &local->postparent, 1);
+                }
 
                 DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
                 DHT_STACK_UNWIND (lookup, frame, local->op_ret,
@@ -825,7 +898,7 @@ dht_lookup_everywhere_done (call_frame_t *frame, xlator_t *this)
                 hashed_subvol->name);
 
         ret = dht_linkfile_create (frame,
-                                   dht_lookup_linkfile_create_cbk,
+                                   dht_lookup_linkfile_create_cbk, this,
                                    cached_subvol, hashed_subvol, &local->loc);
 
         return ret;
@@ -863,8 +936,9 @@ dht_lookup_everywhere_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         xlator_t     *subvol        = NULL;
         loc_t        *loc           = NULL;
         xlator_t     *link_subvol   = NULL;
-        int           ret = -1;
-        int32_t       fd_count = 0;
+        int           ret           = -1;
+        int32_t       fd_count      = 0;
+        dht_conf_t   *conf          = NULL;
 
         GF_VALIDATE_OR_GOTO ("dht", frame, out);
         GF_VALIDATE_OR_GOTO ("dht", this, out);
@@ -874,6 +948,7 @@ dht_lookup_everywhere_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         local  = frame->local;
         loc    = &local->loc;
+        conf   = this->private;
 
         prev   = cookie;
         subvol = prev->this;
@@ -895,7 +970,8 @@ dht_lookup_everywhere_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                 loc->path, prev->this->name);
                 }
 
-                is_linkfile = check_is_linkfile (inode, buf, xattr);
+                is_linkfile = check_is_linkfile (inode, buf, xattr,
+                                                 conf->link_xattr_name);
                 is_dir = check_is_dir (inode, buf, xattr);
 
                 if (is_linkfile) {
@@ -1037,7 +1113,16 @@ dht_lookup_linkfile_cbk (call_frame_t *frame, void *cookie,
                 gf_log (this->name, GF_LOG_INFO,
                         "lookup of %s on %s (following linkfile) failed (%s)",
                         local->loc.path, subvol->name, strerror (op_errno));
-                goto err;
+
+                /* If cached subvol returned ENOTCONN, do not do
+                lookup_everywhere. We need to make sure linkfile does not get
+                removed, which can take away the namespace, and subvol is
+                anyways down. */
+
+                if (op_errno != ENOTCONN)
+                        goto err;
+                else
+                        goto unwind;
         }
 
         if (check_is_dir (inode, stbuf, xattr)) {
@@ -1047,7 +1132,7 @@ dht_lookup_linkfile_cbk (call_frame_t *frame, void *cookie,
                 goto err;
         }
 
-        if (check_is_linkfile (inode, stbuf, xattr)) {
+        if (check_is_linkfile (inode, stbuf, xattr, conf->link_xattr_name)) {
                 gf_log (this->name, GF_LOG_INFO,
                         "lookup of %s on %s (following linkfile) reached link",
                         local->loc.path, subvol->name);
@@ -1075,9 +1160,12 @@ dht_lookup_linkfile_cbk (call_frame_t *frame, void *cookie,
                 op_errno = EINVAL;
         }
 
-unwind:
-        WIPE (postparent);
+        if (local->loc.parent) {
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           postparent, 1);
+        }
 
+unwind:
         DHT_STRIP_PHASE1_FLAGS (stbuf);
         DHT_STACK_UNWIND (lookup, frame, op_ret, op_errno, inode, stbuf, xattr,
                           postparent);
@@ -1159,7 +1247,6 @@ dht_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         loc_t        *loc           = NULL;
         call_frame_t *prev          = NULL;
         int           ret           = 0;
-        uint64_t      tmp_layout    = 0;
         dht_layout_t *parent_layout = NULL;
 
         GF_VALIDATE_OR_GOTO ("dht", frame, err);
@@ -1190,8 +1277,10 @@ dht_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 }
                 if ((conf->search_unhashed == GF_DHT_LOOKUP_UNHASHED_AUTO) &&
                     (loc->parent)) {
-                        ret = inode_ctx_get (loc->parent, this, &tmp_layout);
-                        parent_layout = (dht_layout_t *)(long)tmp_layout;
+                        ret = dht_inode_ctx_layout_get (loc->parent, this,
+                                                        &parent_layout);
+                        if (ret || !parent_layout)
+                                goto out;
                         if (parent_layout->search_unhashed) {
                                 local->op_errno = ENOENT;
                                 dht_lookup_everywhere (frame, this, loc);
@@ -1220,7 +1309,8 @@ dht_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        is_linkfile = check_is_linkfile (inode, stbuf, xattr);
+        is_linkfile = check_is_linkfile (inode, stbuf, xattr,
+                                         conf->link_xattr_name);
 
         if (!is_linkfile) {
                 /* non-directory and not a linkfile */
@@ -1260,7 +1350,10 @@ out:
          * from each of the subvolume. See dht_iatt_merge for reference.
          */
 
-        WIPE (postparent);
+        if (!op_ret && local->loc.parent) {
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           postparent, 1);
+        }
 
         DHT_STRIP_PHASE1_FLAGS (stbuf);
         DHT_STACK_UNWIND (lookup, frame, op_ret, op_errno, inode, stbuf, xattr,
@@ -1322,7 +1415,6 @@ dht_lookup (call_frame_t *frame, xlator_t *this,
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (loc, err);
         VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
 
         conf = this->private;
         if (!conf)
@@ -1366,16 +1458,8 @@ dht_lookup (call_frame_t *frame, xlator_t *this,
                 return 0;
         }
 
-        if (!hashed_subvol) {
+        if (!hashed_subvol)
                 hashed_subvol = dht_subvol_get_hashed (this, loc);
-                if (!hashed_subvol) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to get hashed subvol for %s",
-                                loc->path);
-                        op_errno = EINVAL;
-                        goto err;
-                }
-        }
         local->hashed_subvol = hashed_subvol;
 
         if (is_revalidate (loc)) {
@@ -1405,7 +1489,7 @@ dht_lookup (call_frame_t *frame, xlator_t *this,
                  *       revalidates directly go to the cached-subvolume.
                  */
                 ret = dict_set_uint32 (local->xattr_req,
-                                       "trusted.glusterfs.dht", 4 * 4);
+                                       conf->xattr_name, 4 * 4);
 
                 if (IA_ISDIR (local->inode->ia_type)) {
                         local->call_cnt = call_cnt = conf->subvolume_cnt;
@@ -1440,10 +1524,10 @@ dht_lookup (call_frame_t *frame, xlator_t *this,
         do_fresh_lookup:
                 /* TODO: remove the hard-coding */
                 ret = dict_set_uint32 (local->xattr_req,
-                                       "trusted.glusterfs.dht", 4 * 4);
+                                       conf->xattr_name, 4 * 4);
 
                 ret = dict_set_uint32 (local->xattr_req,
-                                       DHT_LINKFILE_KEY, 256);
+                                       conf->link_xattr_name, 256);
 
                 /* need it for self-healing linkfiles which is
                    'in-migration' state */
@@ -1519,8 +1603,12 @@ dht_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 local->postparent = *postparent;
                 local->preparent = *preparent;
 
-                WIPE (&local->postparent);
-                WIPE (&local->preparent);
+                if (local->loc.parent) {
+                        dht_inode_ctx_time_update (local->loc.parent, this,
+                                                   &local->preparent, 0);
+                        dht_inode_ctx_time_update (local->loc.parent, this,
+                                                   &local->postparent, 1);
+                }
         }
 unlock:
         UNLOCK (&frame->lock);
@@ -1547,7 +1635,8 @@ dht_unlink_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         LOCK (&frame->lock);
         {
-                if (op_ret == -1) {
+                if ((op_ret == -1) && !((op_errno == ENOENT) ||
+                                        (op_errno == ENOTCONN))) {
                         local->op_errno = op_errno;
                         gf_log (this->name, GF_LOG_DEBUG,
                                 "subvolume %s returned -1 (%s)",
@@ -1560,7 +1649,7 @@ dht_unlink_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 unlock:
         UNLOCK (&frame->lock);
 
-        if (op_ret == -1)
+        if (local->op_ret == -1)
                 goto err;
 
         cached_subvol = dht_subvol_get_cached (this, local->loc.inode);
@@ -1584,41 +1673,6 @@ err:
         return 0;
 }
 
-static int
-dht_ufo_xattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int op_ret, int op_errno, dict_t *xdata)
-{
-        dht_local_t  *local = NULL;
-        int           this_call_cnt = 0;
-        call_frame_t *prev = NULL;
-
-        local = frame->local;
-        prev = cookie;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        local->op_ret = -1;
-                        local->op_errno = op_errno;
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "subvolume %s returned -1 (%s)",
-                                prev->this->name, strerror (op_errno));
-                        goto unlock;
-                }
-        }
-unlock:
-        UNLOCK (&frame->lock);
-
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
-                DHT_STACK_UNWIND (setxattr, frame, local->op_ret,
-                                  local->op_errno, NULL);
-        }
-
-        return 0;
-}
-
-
 int
 dht_err_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
              int op_ret, int op_errno, dict_t *xdata)
@@ -1691,115 +1745,225 @@ dht_fill_pathinfo_xattr (xlator_t *this, dht_local_t *local,
 }
 
 int
-dht_vgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
+dht_vgetxattr_alloc_and_fill (dht_local_t *local, dict_t *xattr, xlator_t *this,
+                              int op_errno)
 {
-        dht_local_t *local         = NULL;
-        int          ret           = 0;
-        int          flag          = 0;
-        int          this_call_cnt = 0;
-        char        *value_got     = NULL;
-        char  layout_buf[8192]     = {0,};
-        char        *xattr_buf     = NULL;
-        dict_t      *dict          = NULL;
-        int32_t      alloc_len     = 0;
-        int32_t      plen          = 0;
-        call_frame_t *prev         = NULL;
+        int      ret       = -1;
+        char    *value     = NULL;
+        int32_t  plen      = 0;
 
-        local = frame->local;
-        prev = cookie;
+        ret = dict_get_str (xattr, local->xsel, &value);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Subvolume %s returned -1 (%s)", this->name,
+                        strerror (op_errno));
+                local->op_ret = -1;
+                local->op_errno = op_errno;
+                goto out;
+        }
 
-        if (op_ret >= 0) {
-                ret = dict_get_str (xattr, local->xsel, &value_got);
-                if (!ret) {
-                        alloc_len = strlen (value_got);
+        local->alloc_len += strlen(value);
 
-                        /**
-                         * allocate the buffer:- we allocate 10 bytes extra in
-                         * case we need to append ' Link: ' in the buffer for
-                         * another STACK_WIND
-                         */
+        if (!local->xattr_val) {
+                local->alloc_len += (strlen (DHT_PATHINFO_HEADER) + 10);
+                local->xattr_val = GF_CALLOC (local->alloc_len, sizeof (char),
+                                              gf_common_mt_char);
+                if (!local->xattr_val) {
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        if (local->xattr_val) {
+                plen = strlen (local->xattr_val);
+                if (plen) {
+                        /* extra byte(s) for \0 to be safe */
+                        local->alloc_len += (plen + 2);
+                        local->xattr_val = GF_REALLOC (local->xattr_val,
+                                                       local->alloc_len);
                         if (!local->xattr_val) {
-                                alloc_len += (strlen (DHT_PATHINFO_HEADER) + 10);
-                                local->xattr_val =
-                                        GF_CALLOC (alloc_len,
-                                                   sizeof (char),
-                                                   gf_common_mt_char);
+                                ret = -1;
+                                goto out;
                         }
+                }
 
-                        if (local->xattr_val) {
-                                plen = strlen (local->xattr_val);
-                                if (plen) {
-                                        /* extra byte(s) for \0 to be safe */
-                                        alloc_len += (plen + 2);
-                                        local->xattr_val =
-                                                GF_REALLOC (local->xattr_val,
-                                                            alloc_len);
-                                        if (!local->xattr_val)
-                                                goto out;
-                                }
+                (void) strcat (local->xattr_val, value);
+                (void) strcat (local->xattr_val, " ");
+                local->op_ret = 0;
+        }
 
-                                strcat (local->xattr_val, value_got);
-                        }
-                        local->op_ret = 0;
-                }
+        ret = 0;
+
+ out:
+        return ret;
+}
+
+int
+dht_vgetxattr_fill_and_set (dht_local_t *local, dict_t **dict, xlator_t *this,
+                            gf_boolean_t flag)
+{
+        int   ret             = -1;
+        char *xattr_buf       = NULL;
+        char layout_buf[8192] = {0,};
+
+        if (flag)
+                fill_layout_info (local->layout, layout_buf);
+
+        *dict = dict_new ();
+        if (!*dict)
+                goto out;
+
+        local->xattr_val[strlen (local->xattr_val) - 1] = '\0';
+
+        /* we would need max this many bytes to create xattr string
+         * extra 40 bytes is just an estimated amount of additional
+         * space required as we include translator name and some
+         * spaces, brackets etc. when forming the pathinfo string.
+         *
+         * For node-uuid we just don't have all the pretty formatting,
+         * but since this is a generic routine for pathinfo & node-uuid
+         * we dont have conditional space allocation and try to be
+         * generic
+         */
+        local->alloc_len += (2 * strlen (this->name))
+                + strlen (layout_buf)
+                + 40;
+        xattr_buf = GF_CALLOC (local->alloc_len, sizeof (char),
+                               gf_common_mt_char);
+        if (!xattr_buf)
+                goto out;
+
+        if (XATTR_IS_PATHINFO (local->xsel)) {
+                (void) dht_fill_pathinfo_xattr (this, local, xattr_buf,
+                                                local->alloc_len, flag,
+                                                layout_buf);
+        } else if (XATTR_IS_NODE_UUID (local->xsel)) {
+                (void) snprintf (xattr_buf, local->alloc_len, "%s",
+                                 local->xattr_val);
         } else {
-                local->op_ret = -1;
-                local->op_errno = op_errno;
-                gf_log (this->name, GF_LOG_ERROR, "Subvolume %s returned -1 "
-                        "(%s)", prev->this->name, strerror (op_errno));
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Unknown local->xsel (%s)", local->xsel);
+                GF_FREE (xattr_buf);
+                goto out;
         }
 
+        ret = dict_set_dynstr (*dict, local->xsel, xattr_buf);
+        if (ret)
+                GF_FREE (xattr_buf);
+        GF_FREE (local->xattr_val);
+
  out:
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
+        return ret;
+}
 
-                if (local->op_ret == -1) {
-                        goto unwind;
-                }
+int
+dht_vgetxattr_dir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
+{
+        int          ret           = 0;
+        dht_local_t *local         = NULL;
+        int          this_call_cnt = 0;
+        dict_t      *dict          = NULL;
 
-                if (local->layout->cnt > 1) {
-                        /* Set it for directory */
-                        fill_layout_info (local->layout, layout_buf);
-                        flag = 1;
-                }
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (frame->local, out);
 
-                dict = dict_new ();
-
-                /* we would need max this many bytes to create xattr string */
-                alloc_len += (2 * strlen (this->name))
-                          + strlen (layout_buf)
-                          + 40;
-                xattr_buf = GF_CALLOC (alloc_len,
-                                       sizeof (char), gf_common_mt_char);
-
-                if (XATTR_IS_PATHINFO (local->xsel)) {
-                        (void) dht_fill_pathinfo_xattr (this, local, xattr_buf,
-                                                        alloc_len, flag,
-                                                        layout_buf);
-                } else if (XATTR_IS_NODE_UUID (local->xsel)) {
-                        (void) snprintf (xattr_buf, alloc_len, "%s",
-                                         local->xattr_val);
-                } else
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "Unknown local->xsel (%s)", local->xsel);
+        local = frame->local;
 
-                ret = dict_set_dynstr (dict, local->xsel, xattr_buf);
+        LOCK (&frame->lock);
+        {
+                this_call_cnt = --local->call_cnt;
+                if (op_ret < 0) {
+                        if (op_errno != ENOTCONN) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "getxattr err (%s) for dir",
+                                        strerror (op_errno));
+				local->op_ret = -1;
+				local->op_errno = op_errno;
+                        }
 
-                if (local->xattr_val)
-                        GF_FREE (local->xattr_val);
+                        goto unlock;
+                }
 
-                DHT_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict,
-                                  xdata);
+                ret = dht_vgetxattr_alloc_and_fill (local, xattr, this,
+                                                    op_errno);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "alloc or fill failure");
+        }
+ unlock:
+        UNLOCK (&frame->lock);
 
-                if (dict)
-                        dict_unref (dict);
+        if (!is_last_call (this_call_cnt))
+                goto out;
 
-                return 0;
+        /* -- last call: do patch ups -- */
+
+        if (local->op_ret == -1) {
+                goto unwind;
         }
 
-unwind:
+        ret = dht_vgetxattr_fill_and_set (local, &dict, this, _gf_true);
+        if (ret)
+                goto unwind;
+
+        DHT_STACK_UNWIND (getxattr, frame, 0, 0, dict, xdata);
+        goto cleanup;
+
+ unwind:
         DHT_STACK_UNWIND (getxattr, frame, -1, local->op_errno, NULL, NULL);
+ cleanup:
+        if (dict)
+                dict_unref (dict);
+ out:
+        return 0;
+}
+
+int
+dht_vgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
+{
+        dht_local_t  *local         = NULL;
+        int           ret           = 0;
+        dict_t       *dict          = NULL;
+        call_frame_t *prev          = NULL;
+        gf_boolean_t  flag          = _gf_true;
+
+        local = frame->local;
+        prev = cookie;
+
+        if (op_ret < 0) {
+                local->op_ret = -1;
+                local->op_errno = op_errno;
+                gf_log (this->name, GF_LOG_ERROR, "Subvolume %s returned -1 "
+                        "(%s)", prev->this->name, strerror (op_errno));
+                goto unwind;
+        }
+
+        ret = dht_vgetxattr_alloc_and_fill (local, xattr, this,
+                                            op_errno);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "alloc or fill failure");
+                goto unwind;
+        }
+
+        flag = (local->layout->cnt > 1) ? _gf_true : _gf_false;
+
+        ret = dht_vgetxattr_fill_and_set (local, &dict, this, flag);
+        if (ret)
+                goto unwind;
+
+        DHT_STACK_UNWIND (getxattr, frame, 0, 0, dict, xdata);
+        goto cleanup;
+
+ unwind:
+        DHT_STACK_UNWIND (getxattr, frame, -1, local->op_errno,
+                          NULL, NULL);
+ cleanup:
+        if (dict)
+                dict_unref (dict);
+
         return 0;
 }
 
@@ -1832,10 +1996,13 @@ dht_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 {
         int             this_call_cnt = 0;
         dht_local_t     *local = NULL;
+        dht_conf_t      *conf = NULL;
 
         VALIDATE_OR_GOTO (frame, out);
         VALIDATE_OR_GOTO (frame->local, out);
+        VALIDATE_OR_GOTO (this->private, out);
 
+        conf = this->private;
         local = frame->local;
 
         this_call_cnt = dht_frame_return (frame);
@@ -1843,21 +2010,21 @@ dht_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if (!xattr || (op_ret == -1))
                 goto out;
 
-        if (dict_get (xattr, "trusted.glusterfs.dht")) {
-                dict_del (xattr, "trusted.glusterfs.dht");
+        if (dict_get (xattr, conf->xattr_name)) {
+                dict_del (xattr, conf->xattr_name);
+        }
+
+        if (frame->root->pid >= 0 ) {
+                GF_REMOVE_INTERNAL_XATTR("trusted.glusterfs.quota*", xattr);
+                GF_REMOVE_INTERNAL_XATTR("trusted.pgfid*", xattr);
         }
+
         local->op_ret = 0;
 
         if (!local->xattr) {
                 local->xattr = dict_copy_with_ref (xattr, NULL);
         } else {
-                /* first aggregate everything into xattr and then copy into
-                 * local->xattr. This is required as we want to have
-                 * 'local->xattr' as the proper final dictionary passed above
-                 * distribute xlator.
-                 */
-                dht_aggregate_xattr (xattr, local->xattr);
-                local->xattr = dict_copy (xattr, local->xattr);
+                dht_aggregate_xattr (local->xattr, xattr);
         }
 out:
         if (is_last_call (this_call_cnt)) {
@@ -1877,9 +2044,72 @@ dht_getxattr_unwind (call_frame_t *frame,
 
 
 int
+dht_getxattr_get_real_filename_cbk (call_frame_t *frame, void *cookie,
+				    xlator_t *this, int op_ret, int op_errno,
+				    dict_t *xattr, dict_t *xdata)
+{
+        int             this_call_cnt = 0;
+        dht_local_t     *local = NULL;
+
+
+        local = frame->local;
+
+	if (op_ret != -1) {
+		if (local->xattr)
+			dict_unref (local->xattr);
+		local->xattr = dict_ref (xattr);
+
+		if (local->xattr_req)
+			dict_unref (local->xattr_req);
+		local->xattr_req = dict_ref (xdata);
+	}
+
+	this_call_cnt = dht_frame_return (frame);
+	if (is_last_call (this_call_cnt)) {
+		DHT_STACK_UNWIND (getxattr, frame, local->op_ret, op_errno,
+				  local->xattr, local->xattr_req);
+	}
+
+	return 0;
+}
+
+
+int
+dht_getxattr_get_real_filename (call_frame_t *frame, xlator_t *this,
+				loc_t *loc, const char *key, dict_t *xdata)
+{
+	dht_local_t     *local = NULL;
+	int              i = 0;
+	dht_layout_t    *layout = NULL;
+	int              cnt = 0;
+	xlator_t        *subvol = NULL;
+
+
+	local = frame->local;
+	layout = local->layout;
+
+	cnt = local->call_cnt = layout->cnt;
+
+	local->op_ret = -1;
+	local->op_errno = ENODATA;
+
+	for (i = 0; i < cnt; i++) {
+		subvol = layout->list[i].xlator;
+		STACK_WIND (frame, dht_getxattr_get_real_filename_cbk,
+			    subvol, subvol->fops->getxattr,
+			    loc, key, xdata);
+	}
+
+	return 0;
+}
+
+
+int
 dht_getxattr (call_frame_t *frame, xlator_t *this,
               loc_t *loc, const char *key, dict_t *xdata)
+#define DHT_IS_DIR(layout)  (layout->cnt > 1)
 {
+
         xlator_t     *subvol        = NULL;
         xlator_t     *hashed_subvol = NULL;
         xlator_t     *cached_subvol = NULL;
@@ -1895,7 +2125,6 @@ dht_getxattr (call_frame_t *frame, xlator_t *this,
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (loc, err);
         VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
         VALIDATE_OR_GOTO (this->private, err);
 
         conf   = this->private;
@@ -1923,8 +2152,40 @@ dht_getxattr (call_frame_t *frame, xlator_t *this,
                 }
         }
 
-        if (key && ((strcmp (key, GF_XATTR_PATHINFO_KEY) == 0)
-                    || strcmp (key, GF_XATTR_NODE_UUID_KEY) == 0)) {
+	if (key &&
+	    (strncmp (key, GF_XATTR_GET_REAL_FILENAME_KEY,
+		      strlen (GF_XATTR_GET_REAL_FILENAME_KEY)) == 0)
+	    && DHT_IS_DIR(layout)) {
+		dht_getxattr_get_real_filename (frame, this, loc, key, xdata);
+		return 0;
+	}
+
+        /* for file use cached subvolume (obviously!): see if {}
+         * below
+         * for directory:
+         *  wind to all subvolumes and exclude subvolumes which
+         *  return ENOTCONN (in callback)
+         *
+         * NOTE: Don't trust inode here, as that may not be valid
+         *       (until inode_link() happens)
+         */
+        if (key && DHT_IS_DIR(layout) &&
+            (XATTR_IS_PATHINFO (key)
+             || (strcmp (key, GF_XATTR_NODE_UUID_KEY) == 0))) {
+                (void) strncpy (local->xsel, key, 256);
+                cnt = local->call_cnt = layout->cnt;
+                for (i = 0; i < cnt; i++) {
+                        subvol = layout->list[i].xlator;
+                        STACK_WIND (frame, dht_vgetxattr_dir_cbk,
+                                    subvol, subvol->fops->getxattr,
+                                    loc, key, NULL);
+                }
+                return 0;
+        }
+
+        /* node-uuid or pathinfo for files */
+        if (key && ((strcmp (key, GF_XATTR_NODE_UUID_KEY) == 0)
+                    || XATTR_IS_PATHINFO (key))) {
                 cached_subvol = local->cached_subvol;
                 (void) strncpy (local->xsel, key, 256);
 
@@ -1934,6 +2195,7 @@ dht_getxattr (call_frame_t *frame, xlator_t *this,
 
                 return 0;
         }
+
         if (key && (strcmp (key, GF_XATTR_LINKINFO_KEY) == 0)) {
                 hashed_subvol = dht_subvol_get_hashed (this, loc);
                 if (!hashed_subvol) {
@@ -1966,13 +2228,13 @@ dht_getxattr (call_frame_t *frame, xlator_t *this,
         }
 
         if (key && (!strcmp (GF_XATTR_MARKER_KEY, key))
-            && (-1 == frame->root->pid)) {
-
-                if (loc->inode-> ia_type == IA_IFDIR) {
+            && (GF_CLIENT_PID_GSYNCD == frame->root->pid)) {
+                if (DHT_IS_DIR(layout)) {
                         cnt = layout->cnt;
                 } else {
                         cnt = 1;
                 }
+
                 sub_volumes = alloca ( cnt * sizeof (xlator_t *));
                 for (i = 0; i < cnt; i++)
                         *(sub_volumes + i) = layout->list[i].xlator;
@@ -1980,7 +2242,8 @@ dht_getxattr (call_frame_t *frame, xlator_t *this,
                 if (cluster_getmarkerattr (frame, this, loc, key,
                                            local, dht_getxattr_unwind,
                                            sub_volumes, cnt,
-                                           MARKER_UUID_TYPE, conf->vol_uuid)) {
+                                           MARKER_UUID_TYPE, marker_uuid_default_gauge,
+                                           conf->vol_uuid)) {
                         op_errno = EINVAL;
                         goto err;
                 }
@@ -1988,10 +2251,22 @@ dht_getxattr (call_frame_t *frame, xlator_t *this,
                 return 0;
         }
 
+        if (key && !strcmp (GF_XATTR_QUOTA_LIMIT_LIST, key)) {
+                /* quota hardlimit and aggregated size of a directory is stored
+                 * in inode contexts of each brick. Hence its good enough that
+                 * we send getxattr for this key to any brick.
+                 */
+                local->call_cnt = 1;
+                subvol = dht_first_up_subvol (this);
+                STACK_WIND (frame, dht_getxattr_cbk, subvol,
+                            subvol->fops->getxattr, loc, key, xdata);
+                return 0;
+        }
+
         if (key && *conf->vol_uuid) {
                 if ((match_uuid_local (key, conf->vol_uuid) == 0) &&
-                    (-1 == frame->root->pid)) {
-                        if (loc->inode-> ia_type == IA_IFDIR) {
+                    (GF_CLIENT_PID_GSYNCD == frame->root->pid)) {
+                        if (DHT_IS_DIR(layout)) {
                                 cnt = layout->cnt;
                         } else {
                                 cnt = 1;
@@ -2004,6 +2279,7 @@ dht_getxattr (call_frame_t *frame, xlator_t *this,
                                                    local, dht_getxattr_unwind,
                                                    sub_volumes, cnt,
                                                    MARKER_XTIME_TYPE,
+                                                   marker_xtime_default_gauge,
                                                    conf->vol_uuid)) {
                                 op_errno = EINVAL;
                                 goto err;
@@ -2013,7 +2289,7 @@ dht_getxattr (call_frame_t *frame, xlator_t *this,
                 }
         }
 
-        if (loc->inode-> ia_type == IA_IFDIR) {
+        if (DHT_IS_DIR(layout)) {
                 cnt = local->call_cnt = layout->cnt;
         } else {
                 cnt = local->call_cnt  = 1;
@@ -2033,6 +2309,7 @@ err:
 
         return 0;
 }
+#undef DHT_IS_DIR
 
 int
 dht_fgetxattr (call_frame_t *frame, xlator_t *this,
@@ -2074,7 +2351,10 @@ dht_fgetxattr (call_frame_t *frame, xlator_t *this,
                 }
         }
 
-        if (fd->inode->ia_type == IA_IFDIR) {
+        if ((fd->inode->ia_type == IA_IFDIR)
+	    && key
+            && (strncmp (key, GF_XATTR_LOCKINFO_KEY,
+                         strlen (GF_XATTR_LOCKINFO_KEY) != 0))) {
                 cnt = local->call_cnt = layout->cnt;
         } else {
                 cnt = local->call_cnt  = 1;
@@ -2102,15 +2382,18 @@ dht_fsetxattr (call_frame_t *frame, xlator_t *this,
         xlator_t     *subvol   = NULL;
         dht_local_t  *local    = NULL;
         int           op_errno = EINVAL;
-        data_pair_t  *trav     = NULL;
+        dht_conf_t   *conf     = NULL;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (fd, err);
         VALIDATE_OR_GOTO (fd->inode, err);
+        VALIDATE_OR_GOTO (this->private, err);
+
+        conf = this->private;
 
-        GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.dht*", xattr,
-                                   trav, op_errno, err);
+        GF_IF_INTERNAL_XATTR_GOTO (conf->wild_xattr_name, xattr,
+                                   op_errno, err);
 
         local = dht_local_init (frame, NULL, fd, GF_FOP_FSETXATTR);
         if (!local) {
@@ -2208,19 +2491,17 @@ dht_setxattr (call_frame_t *frame, xlator_t *this,
         char          value[4096] = {0,};
         gf_dht_migrate_data_type_t forced_rebalance = GF_DHT_MIGRATE_DATA;
         int           call_cnt = 0;
-        data_pair_t  *trav     = NULL;
-
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (loc, err);
         VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-        GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.dht*", xattr,
-                                   trav, op_errno, err);
 
         conf   = this->private;
+
+        GF_IF_INTERNAL_XATTR_GOTO (conf->wild_xattr_name, xattr,
+                                   op_errno, err);
+
         local = dht_local_init (frame, loc, NULL, GF_FOP_SETXATTR);
         if (!local) {
                 op_errno = ENOMEM;
@@ -2245,25 +2526,6 @@ dht_setxattr (call_frame_t *frame, xlator_t *this,
 
         local->call_cnt = call_cnt = layout->cnt;
 
-        /*  This key is sent by Unified File and Object storage
-         *  to test xattr support in backend.
-         */
-        tmp = dict_get (xattr, "user.ufo-test");
-        if (tmp) {
-                if (IA_ISREG (loc->inode->ia_type)) {
-                        op_errno = ENOTSUP;
-                        goto err;
-                }
-                local->op_ret = 0;
-                for (i = 0; i < call_cnt; i++) {
-                        STACK_WIND (frame, dht_ufo_xattr_cbk,
-                                    layout->list[i].xlator,
-                                    layout->list[i].xlator->fops->setxattr,
-                                    loc, xattr, flags, NULL);
-                }
-                return 0;
-        }
-
         tmp = dict_get (xattr, "distribute.migrate-data");
         if (tmp) {
                 if (IA_ISDIR (loc->inode->ia_type)) {
@@ -2338,9 +2600,13 @@ dht_setxattr (call_frame_t *frame, xlator_t *this,
                 gf_log (this->name, GF_LOG_INFO,
                         "fixing the layout of %s", loc->path);
 
-                dht_fix_directory_layout (frame, dht_common_setxattr_cbk,
-                                          layout);
-                return 0;
+                ret = dht_fix_directory_layout (frame, dht_common_setxattr_cbk,
+                                                layout);
+                if (ret) {
+                        op_errno = ENOTCONN;
+                        goto err;
+                }
+                return ret;
         }
 
         tmp = dict_get (xattr, "distribute.directory-spread-count");
@@ -2352,10 +2618,14 @@ dht_setxattr (call_frame_t *frame, xlator_t *this,
                              (dir_spread > 0))) {
                         layout->spread_cnt = dir_spread;
 
-                        dht_fix_directory_layout (frame,
-                                                  dht_common_setxattr_cbk,
-                                                  layout);
-                        return 0;
+                        ret = dht_fix_directory_layout (frame,
+                                                        dht_common_setxattr_cbk,
+                                                        layout);
+                        if (ret) {
+                                op_errno = ENOTCONN;
+                                goto err;
+                        }
+                        return ret;
                 }
                 gf_log (this->name, GF_LOG_ERROR,
                         "wrong 'directory-spread-count' value (%s)", value);
@@ -2425,18 +2695,20 @@ dht_removexattr (call_frame_t *frame, xlator_t *this,
         dht_local_t  *local = NULL;
         dht_layout_t *layout = NULL;
         int           call_cnt = 0;
+        dht_conf_t   *conf = NULL;
 
         int i;
 
         VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (this->private, err);
+
+        conf = this->private;
 
-        GF_IF_NATIVE_XATTR_GOTO ("trusted.glusterfs.dht*",
-                                 key, op_errno, err);
+        GF_IF_NATIVE_XATTR_GOTO (conf->wild_xattr_name, key, op_errno, err);
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (loc, err);
         VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
 
         local = dht_local_init (frame, loc, NULL, GF_FOP_REMOVEXATTR);
         if (!local) {
@@ -2488,13 +2760,16 @@ dht_fremovexattr (call_frame_t *frame, xlator_t *this,
         dht_local_t  *local = NULL;
         dht_layout_t *layout = NULL;
         int           call_cnt = 0;
+        dht_conf_t   *conf = 0;
 
         int i;
 
         VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (this->private, err);
+
+        conf = this->private;
 
-        GF_IF_NATIVE_XATTR_GOTO ("trusted.glusterfs.dht*",
-                                 key, op_errno, err);
+        GF_IF_NATIVE_XATTR_GOTO (conf->wild_xattr_name, key, op_errno, err);
 
         VALIDATE_OR_GOTO (frame, err);
 
@@ -2602,11 +2877,16 @@ int
 dht_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 int op_ret, int op_errno, struct statvfs *statvfs, dict_t *xdata)
 {
-        dht_local_t *local         = NULL;
-        int          this_call_cnt = 0;
-        int          bsize         = 0;
-        int          frsize        = 0;
+        dht_local_t *local              = NULL;
+        int          this_call_cnt      = 0;
+        int          bsize              = 0;
+        int          frsize             = 0;
+        int8_t       quota_deem_statfs  = 0;
+        GF_UNUSED int     ret           = 0;
+        unsigned long     new_usage     = 0;
+        unsigned long     cur_usage     = 0;
 
+        ret = dict_get_int8 (xdata, "quota-deem-statfs", &quota_deem_statfs);
 
         local = frame->local;
 
@@ -2616,8 +2896,22 @@ dht_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->op_errno = op_errno;
                         goto unlock;
                 }
+                if (!statvfs) {
+                        op_errno = EINVAL;
+                        local->op_ret = -1;
+                        goto unlock;
+                }
                 local->op_ret = 0;
 
+                if (quota_deem_statfs) {
+                        new_usage = statvfs->f_blocks - statvfs->f_bfree;
+                        cur_usage = local->statvfs.f_blocks - local->statvfs.f_bfree;
+                        /* We take the maximux of the usage from the subvols */
+                        if (new_usage >= cur_usage)
+                                local->statvfs = *statvfs;
+                        goto unlock;
+                }
+
                 if (local->statvfs.f_bsize != 0) {
                         bsize = max(local->statvfs.f_bsize, statvfs->f_bsize);
                         frsize = max(local->statvfs.f_frsize, statvfs->f_frsize);
@@ -2638,6 +2932,7 @@ dht_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 local->statvfs.f_flag     = statvfs->f_flag;
                 local->statvfs.f_namemax  = statvfs->f_namemax;
 
+
         }
 unlock:
         UNLOCK (&frame->lock);
@@ -2665,7 +2960,6 @@ dht_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (loc, err);
         VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
         VALIDATE_OR_GOTO (this->private, err);
 
         conf = this->private;
@@ -2768,7 +3062,7 @@ dht_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
         dht_layout_t *layout = 0;
         dht_conf_t   *conf   = NULL;
         xlator_t     *subvol = 0;
-        int           ret = 0;
+        int           ret    = 0;
 
         INIT_LIST_HEAD (&entries.list);
         prev = cookie;
@@ -2785,10 +3079,13 @@ dht_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
 
         list_for_each_entry (orig_entry, (&orig_entries->list), list) {
                 next_offset = orig_entry->d_off;
-                if ((check_is_dir (NULL, (&orig_entry->d_stat), NULL) &&
-                     (prev->this != dht_first_up_subvol (this))) ||
-                    check_is_linkfile (NULL, (&orig_entry->d_stat),
-                                       orig_entry->dict)) {
+                if (check_is_dir (NULL, (&orig_entry->d_stat), NULL) &&
+                    (prev->this != local->first_up_subvol)) {
+                        continue;
+                }
+                if (check_is_linkfile (NULL, (&orig_entry->d_stat),
+                                       orig_entry->dict,
+                                       conf->link_xattr_name)) {
                         continue;
                 }
 
@@ -2817,6 +3114,24 @@ dht_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
                 entry->d_type = orig_entry->d_type;
                 entry->d_len  = orig_entry->d_len;
 
+                if (orig_entry->dict)
+                        entry->dict = dict_ref (orig_entry->dict);
+
+                /* making sure we set the inode ctx right with layout,
+                   currently possible only for non-directories, so for
+                   directories don't set entry inodes */
+                if (!IA_ISDIR(entry->d_stat.ia_type) && orig_entry->inode) {
+                        ret = dht_layout_preset (this, prev->this,
+                                                 orig_entry->inode);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "failed to link the layout in inode");
+                        entry->inode = inode_ref (orig_entry->inode);
+                } else if (orig_entry->inode) {
+                        dht_inode_ctx_time_update (orig_entry->inode, this,
+                                                   &entry->d_stat, 1);
+                }
+
                 list_add_tail (&entry->list, &entries.list);
                 count++;
         }
@@ -2847,13 +3162,16 @@ done:
                 }
 
 		if (conf->readdir_optimize == _gf_true) {
-                        if (next_subvol != dht_first_up_subvol (this)) {
+                        if (next_subvol != local->first_up_subvol) {
                                 ret = dict_set_int32 (local->xattr,
                                                       GF_READDIR_SKIP_DIRS, 1);
                                 if (ret)
                                         gf_log (this->name, GF_LOG_ERROR,
 					         "dict set failed");
-		        }
+		        } else {
+                                 dict_del (local->xattr,
+                                           GF_READDIR_SKIP_DIRS);
+                        }
                 }
 
                 STACK_WIND (frame, dht_readdirp_cbk,
@@ -2986,6 +3304,7 @@ dht_do_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (fd, err);
+        VALIDATE_OR_GOTO (this->private, err);
 
         conf = this->private;
 
@@ -2998,6 +3317,7 @@ dht_do_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
         local->fd = fd_ref (fd);
         local->size = size;
         local->xattr_req = (dict)? dict_ref (dict) : NULL;
+        local->first_up_subvol = dht_first_up_subvol (this);
 
         dht_deitransform (this, yoff, &xvol, (uint64_t *)&xoff);
 
@@ -3010,20 +3330,22 @@ dht_do_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
 
                 if (local->xattr) {
                         ret = dict_set_uint32 (local->xattr,
-                                               "trusted.glusterfs.dht.linkto",
-                                               256);
+                                               conf->link_xattr_name, 256);
                         if (ret)
                                 gf_log (this->name, GF_LOG_WARNING,
-                                        "failed to set 'glusterfs.dht.linkto'"
-                                        " key");
+                                        "failed to set '%s' key",
+                                        conf->link_xattr_name);
 			if (conf->readdir_optimize == _gf_true) {
-                                if (xvol != dht_first_up_subvol (this)) {
+                                if (xvol != local->first_up_subvol) {
 				        ret = dict_set_int32 (local->xattr,
 			                               GF_READDIR_SKIP_DIRS, 1);
 				        if (ret)
 					        gf_log (this->name,
                                                         GF_LOG_ERROR,
 						        "Dict set failed");
+                                } else {
+                                        dict_del (local->xattr,
+                                                  GF_READDIR_SKIP_DIRS);
                                 }
 			}
                 }
@@ -3159,7 +3481,7 @@ dht_newfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  inode_t *inode, struct iatt *stbuf, struct iatt *preparent,
                  struct iatt *postparent, dict_t *xdata)
 {
-        call_frame_t *prev = NULL;
+        xlator_t     *prev = NULL;
         int           ret = -1;
         dht_local_t  *local = NULL;
 
@@ -3177,15 +3499,18 @@ dht_newfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         prev = cookie;
 
         if (local->loc.parent) {
-                WIPE (preparent);
-                WIPE (postparent);
+
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           preparent, 0);
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           postparent, 1);
         }
 
-        ret = dht_layout_preset (this, prev->this, inode);
+        ret = dht_layout_preset (this, prev, inode);
         if (ret < 0) {
                 gf_log (this->name, GF_LOG_DEBUG,
                         "could not set pre-set layout for subvolume %s",
-                        prev->this->name);
+                        prev? prev->name: NULL);
                 op_ret   = -1;
                 op_errno = EINVAL;
                 goto out;
@@ -3221,12 +3546,17 @@ dht_mknod_linkfile_create_cbk (call_frame_t *frame, void *cookie,
                 goto err;
 
         local = frame->local;
+        if (!local || !local->cached_subvol) {
+                op_errno = EINVAL;
+                goto err;
+        }
+
         cached_subvol = local->cached_subvol;
 
-        STACK_WIND (frame, dht_newfile_cbk,
-                    cached_subvol, cached_subvol->fops->mknod,
-                    &local->loc, local->mode, local->rdev, local->umask,
-                    local->params);
+        STACK_WIND_COOKIE (frame, dht_newfile_cbk, (void *)cached_subvol,
+                           cached_subvol, cached_subvol->fops->mknod,
+                           &local->loc, local->mode, local->rdev, local->umask,
+                           local->params);
 
         return 0;
 err:
@@ -3269,11 +3599,13 @@ dht_mknod (call_frame_t *frame, xlator_t *this,
                 gf_log (this->name, GF_LOG_TRACE,
                         "creating %s on %s", loc->path, subvol->name);
 
-                STACK_WIND (frame, dht_newfile_cbk,
-                            subvol, subvol->fops->mknod,
-                            loc, mode, rdev, umask, params);
+                STACK_WIND_COOKIE (frame, dht_newfile_cbk, (void *)subvol,
+                                   subvol, subvol->fops->mknod, loc, mode,
+                                   rdev, umask, params);
         } else {
-                avail_subvol = dht_free_disk_available_subvol (this, subvol);
+
+                avail_subvol = dht_free_disk_available_subvol (this, subvol,
+                                                               local);
                 if (avail_subvol != subvol) {
                         /* Choose the minimum filled volume, and create the
                            files there */
@@ -3285,14 +3617,15 @@ dht_mknod (call_frame_t *frame, xlator_t *this,
                         local->umask = umask;
                         dht_linkfile_create (frame,
                                              dht_mknod_linkfile_create_cbk,
-                                             avail_subvol, subvol, loc);
+                                             this, avail_subvol, subvol, loc);
                 } else {
                         gf_log (this->name, GF_LOG_TRACE,
                                 "creating %s on %s", loc->path, subvol->name);
 
-                        STACK_WIND (frame, dht_newfile_cbk,
-                                    subvol, subvol->fops->mknod,
-                                    loc, mode, rdev, umask, params);
+                        STACK_WIND_COOKIE (frame, dht_newfile_cbk,
+                                           (void *)subvol, subvol,
+                                           subvol->fops->mknod, loc, mode,
+                                           rdev, umask, params);
                 }
         }
 
@@ -3337,9 +3670,9 @@ dht_symlink (call_frame_t *frame, xlator_t *this,
         gf_log (this->name, GF_LOG_TRACE,
                 "creating %s on %s", loc->path, subvol->name);
 
-        STACK_WIND (frame, dht_newfile_cbk,
-                    subvol, subvol->fops->symlink,
-                    linkname, loc, umask, params);
+        STACK_WIND_COOKIE (frame, dht_newfile_cbk, (void *)subvol, subvol,
+                           subvol->fops->symlink, linkname, loc, umask,
+                           params);
 
         return 0;
 
@@ -3384,12 +3717,12 @@ dht_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
         }
 
         hashed_subvol = dht_subvol_get_hashed (this, loc);
+        /* Dont fail unlink if hashed_subvol is NULL which can be the result
+         * of layout anomaly */
         if (!hashed_subvol) {
                 gf_log (this->name, GF_LOG_DEBUG,
                         "no subvolume in layout for path=%s",
                         loc->path);
-                op_errno = EINVAL;
-                goto err;
         }
 
         cached_subvol = local->cached_subvol;
@@ -3401,7 +3734,7 @@ dht_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
         }
 
         local->flags = xflag;
-        if (hashed_subvol != cached_subvol) {
+        if (hashed_subvol && hashed_subvol != cached_subvol) {
                 STACK_WIND (frame, dht_unlink_linkfile_cbk,
                             hashed_subvol, hashed_subvol->fops->unlink, loc,
                             xflag, xdata);
@@ -3428,9 +3761,12 @@ dht_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 {
         call_frame_t *prev = NULL;
         dht_layout_t *layout = NULL;
+        dht_local_t  *local = NULL;
 
         prev = cookie;
 
+        local = frame->local;
+
         if (op_ret == -1)
                 goto out;
 
@@ -3444,9 +3780,16 @@ dht_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-        WIPE (preparent);
-        WIPE (postparent);
-
+        if (local->loc.parent) {
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           preparent, 0);
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           postparent, 1);
+        }
+        if (local->linked == _gf_true) {
+                local->stbuf = *stbuf;
+                dht_linkfile_attr_heal (frame, this);
+        }
 out:
         DHT_STRIP_PHASE1_FLAGS (stbuf);
         DHT_STACK_UNWIND (link, frame, op_ret, op_errno, inode, stbuf, preparent,
@@ -3533,7 +3876,7 @@ dht_link (call_frame_t *frame, xlator_t *this,
 
         if (hashed_subvol != cached_subvol) {
                 uuid_copy (local->gfid, oldloc->inode->gfid);
-                dht_linkfile_create (frame, dht_link_linkfile_cbk,
+                dht_linkfile_create (frame, dht_link_linkfile_cbk, this,
                                      cached_subvol, hashed_subvol, newloc);
         } else {
                 STACK_WIND (frame, dht_link_cbk,
@@ -3574,8 +3917,11 @@ dht_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         prev = cookie;
 
         if (local->loc.parent) {
-                WIPE (preparent);
-                WIPE (postparent);
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           preparent, 0);
+
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           postparent, 1);
         }
 
         ret = dht_layout_preset (this, prev->this, inode);
@@ -3680,7 +4026,7 @@ dht_create (call_frame_t *frame, xlator_t *this,
         }
         /* Choose the minimum filled volume, and create the
            files there */
-        avail_subvol = dht_free_disk_available_subvol (this, subvol);
+        avail_subvol = dht_free_disk_available_subvol (this, subvol, local);
         if (avail_subvol != subvol) {
                 local->params = dict_ref (params);
                 local->flags = flags;
@@ -3691,9 +4037,8 @@ dht_create (call_frame_t *frame, xlator_t *this,
                 gf_log (this->name, GF_LOG_TRACE,
                         "creating %s on %s (link at %s)", loc->path,
                         avail_subvol->name, subvol->name);
-                dht_linkfile_create (frame,
-                                     dht_create_linkfile_create_cbk,
-                                     avail_subvol, subvol, loc);
+                dht_linkfile_create (frame, dht_create_linkfile_create_cbk,
+                                     this, avail_subvol, subvol, loc);
                 goto done;
         }
         gf_log (this->name, GF_LOG_TRACE,
@@ -3727,8 +4072,11 @@ dht_mkdir_selfheal_cbk (call_frame_t *frame, void *cookie,
         if (op_ret == 0) {
                 dht_layout_set (this, local->inode, layout);
                 if (local->loc.parent) {
-                        WIPE (&local->preparent);
-                        WIPE (&local->postparent);
+                        dht_inode_ctx_time_update (local->loc.parent, this,
+                                                   &local->preparent, 0);
+
+                        dht_inode_ctx_time_update (local->loc.parent, this,
+                                                   &local->postparent, 1);
                 }
         }
 
@@ -3763,6 +4111,15 @@ dht_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         ret = dht_layout_merge (this, layout, prev->this,
                                                 -1, ENOSPC, NULL);
                 } else {
+			if (op_ret == -1 && op_errno == EEXIST)
+				/* Very likely just a race between mkdir and
+				   self-heal (from lookup of a concurrent mkdir
+				   attempt).
+				   Ignore error for now. layout setting will
+				   anyways fail if this was a different (old)
+				   pre-existing different directory.
+				*/
+				op_ret = 0;
                         ret = dht_layout_merge (this, layout, prev->this,
                                                 op_ret, op_errno, NULL);
                 }
@@ -3995,8 +4352,15 @@ unlock:
                } else {
 
                         if (local->loc.parent) {
-                                WIPE (&local->preparent);
-                                WIPE (&local->postparent);
+                                dht_inode_ctx_time_update (local->loc.parent,
+                                                           this,
+                                                           &local->preparent,
+                                                           0);
+
+                                dht_inode_ctx_time_update (local->loc.parent,
+                                                           this,
+                                                           &local->postparent,
+                                                           1);
                         }
 
                         DHT_STACK_UNWIND (rmdir, frame, local->op_ret,
@@ -4082,8 +4446,17 @@ unlock:
                         /* All subvol's have responded, proceed */
 
                         if (local->loc.parent) {
-                                WIPE (&local->preparent);
-                                WIPE (&local->postparent);
+
+                                dht_inode_ctx_time_update (local->loc.parent,
+                                                           this,
+                                                           &local->preparent,
+                                                           0);
+
+                                dht_inode_ctx_time_update (local->loc.parent,
+                                                           this,
+                                                           &local->postparent,
+                                                           1);
+
                         }
 
                         DHT_STACK_UNWIND (rmdir, frame, local->op_ret,
@@ -4204,6 +4577,7 @@ dht_rmdir_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         call_frame_t   *main_frame = NULL;
         dht_local_t    *main_local = NULL;
         int             this_call_cnt = 0;
+        dht_conf_t     *conf = this->private;
 
         local = frame->local;
         prev  = cookie;
@@ -4215,7 +4589,7 @@ dht_rmdir_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if (op_ret != 0)
                 goto err;
 
-        if (check_is_linkfile (inode, stbuf, xattr) == 0) {
+        if (!check_is_linkfile (inode, stbuf, xattr, conf->link_xattr_name)) {
                 main_local->op_ret  = -1;
                 main_local->op_errno = ENOTEMPTY;
 
@@ -4240,16 +4614,85 @@ err:
 
 
 int
+dht_rmdir_cached_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                             int op_ret, int op_errno, inode_t *inode,
+                             struct iatt *stbuf, dict_t *xattr,
+                             struct iatt *parent)
+{
+        dht_local_t    *local         = NULL;
+        xlator_t       *src           = NULL;
+        call_frame_t   *main_frame    = NULL;
+        dht_local_t    *main_local    = NULL;
+        int             this_call_cnt = 0;
+        dht_conf_t     *conf          = this->private;
+        dict_t         *xattrs        = NULL;
+        int             ret           = 0;
+
+        local = frame->local;
+        src   = local->hashed_subvol;
+
+        main_frame = local->main_frame;
+        main_local = main_frame->local;
+
+        if (op_ret == 0) {
+                main_local->op_ret  = -1;
+                main_local->op_errno = ENOTEMPTY;
+
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s found on cached subvol %s",
+                        local->loc.path, src->name);
+                goto err;
+        } else if (op_errno != ENOENT) {
+                main_local->op_ret  = -1;
+                main_local->op_errno = op_errno;
+                goto err;
+        }
+
+        xattrs = dict_new ();
+        if (!xattrs) {
+                gf_log (this->name, GF_LOG_ERROR, "dict_new failed");
+                goto err;
+        }
+
+        ret = dict_set_uint32 (xattrs, conf->link_xattr_name, 256);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "failed to set linkto key"
+                        " in dict");
+                if (xattrs)
+                        dict_unref (xattrs);
+                goto err;
+        }
+
+        STACK_WIND (frame, dht_rmdir_lookup_cbk,
+                    src, src->fops->lookup, &local->loc, xattrs);
+        if (xattrs)
+                dict_unref (xattrs);
+
+        return 0;
+err:
+
+        this_call_cnt = dht_frame_return (main_frame);
+        if (is_last_call (this_call_cnt))
+                dht_rmdir_do (main_frame, this);
+
+        DHT_STACK_DESTROY (frame);
+        return 0;
+}
+
+
+int
 dht_rmdir_is_subvol_empty (call_frame_t *frame, xlator_t *this,
                            gf_dirent_t *entries, xlator_t *src)
 {
-        int                 ret = 0;
-        int                 build_ret = 0;
-        gf_dirent_t        *trav = NULL;
+        int                 ret          = 0;
+        int                 build_ret    = 0;
+        gf_dirent_t        *trav         = NULL;
         call_frame_t       *lookup_frame = NULL;
         dht_local_t        *lookup_local = NULL;
-        dht_local_t        *local = NULL;
-        dict_t             *xattrs = NULL;
+        dht_local_t        *local        = NULL;
+        dict_t             *xattrs       = NULL;
+        dht_conf_t         *conf         = this->private;
+        xlator_t           *subvol       = NULL;
 
         local = frame->local;
 
@@ -4258,7 +4701,8 @@ dht_rmdir_is_subvol_empty (call_frame_t *frame, xlator_t *this,
                         continue;
                 if (strcmp (trav->d_name, "..") == 0)
                         continue;
-                if (check_is_linkfile (NULL, (&trav->d_stat), trav->dict)) {
+                if (check_is_linkfile (NULL, (&trav->d_stat), trav->dict,
+                                              conf->link_xattr_name)) {
                         ret++;
                         continue;
                 }
@@ -4276,7 +4720,7 @@ dht_rmdir_is_subvol_empty (call_frame_t *frame, xlator_t *this,
                 return -1;
         }
 
-        ret = dict_set_uint32 (xattrs, DHT_LINKFILE_KEY, 256);
+        ret = dict_set_uint32 (xattrs, conf->link_xattr_name, 256);
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR, "failed to set linkto key"
                         " in dict");
@@ -4308,6 +4752,7 @@ dht_rmdir_is_subvol_empty (call_frame_t *frame, xlator_t *this,
 
                 lookup_frame->local = lookup_local;
                 lookup_local->main_frame = frame;
+                lookup_local->hashed_subvol = src;
 
                 build_ret = dht_build_child_loc (this, &lookup_local->loc,
                                                  &local->loc, trav->d_name);
@@ -4326,9 +4771,20 @@ dht_rmdir_is_subvol_empty (call_frame_t *frame, xlator_t *this,
                 }
                 UNLOCK (&frame->lock);
 
-                STACK_WIND (lookup_frame, dht_rmdir_lookup_cbk,
-                            src, src->fops->lookup,
-                            &lookup_local->loc, xattrs);
+                subvol = dht_linkfile_subvol (this, NULL, &trav->d_stat,
+                                              trav->dict);
+                if (!subvol) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "linkfile not having link subvolume. path=%s",
+                                lookup_local->loc.path);
+                        STACK_WIND (lookup_frame, dht_rmdir_lookup_cbk,
+                                    src, src->fops->lookup,
+                                    &lookup_local->loc, xattrs);
+                } else {
+                        STACK_WIND (lookup_frame, dht_rmdir_cached_lookup_cbk,
+                                    subvol, subvol->fops->lookup,
+                                    &lookup_local->loc, xattrs);
+                }
                 ret++;
         }
 
@@ -4394,15 +4850,18 @@ int
 dht_rmdir_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                        int op_ret, int op_errno, fd_t *fd, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
+        dht_local_t  *local         = NULL;
         int           this_call_cnt = -1;
-        call_frame_t *prev = NULL;
-        dict_t       *dict = NULL;
-        int           ret = 0;
+        call_frame_t *prev          = NULL;
+        dict_t       *dict          = NULL;
+        int           ret           = 0;
+        dht_conf_t   *conf          = this->private;
+        int           i             = 0;
 
         local = frame->local;
         prev  = cookie;
 
+        this_call_cnt = dht_frame_return (frame);
         if (op_ret == -1) {
                 gf_log (this->name, GF_LOG_DEBUG,
                         "opendir on %s for %s failed (%s)",
@@ -4415,6 +4874,12 @@ dht_rmdir_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto err;
         }
 
+        if (!is_last_call (this_call_cnt))
+                return 0;
+
+        if (local->op_ret == -1)
+                goto err;
+
         dict = dict_new ();
         if (!dict) {
                 local->op_ret = -1;
@@ -4422,16 +4887,19 @@ dht_rmdir_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto err;
         }
 
-        ret = dict_set_uint32 (dict,
-                               "trusted.glusterfs.dht.linkto", 256);
+        ret = dict_set_uint32 (dict, conf->link_xattr_name, 256);
         if (ret)
                 gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to set 'trusted.glusterfs.dht.linkto' key",
-                        local->loc.path);
+                        "%s: failed to set '%s' key",
+                        local->loc.path, conf->link_xattr_name);
 
-        STACK_WIND (frame, dht_rmdir_readdirp_cbk,
-                    prev->this, prev->this->fops->readdirp,
-                    local->fd, 4096, 0, dict);
+        local->call_cnt = conf->subvolume_cnt;
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                STACK_WIND (frame, dht_rmdir_readdirp_cbk,
+                            conf->subvolumes[i],
+                            conf->subvolumes[i]->fops->readdirp,
+                            local->fd, 4096, 0, dict);
+        }
 
         if (dict)
                 dict_unref (dict);
@@ -4439,8 +4907,6 @@ dht_rmdir_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         return 0;
 
 err:
-        this_call_cnt = dht_frame_return (frame);
-
         if (is_last_call (this_call_cnt)) {
                 dht_rmdir_do (frame, this);
         }
@@ -4526,7 +4992,6 @@ dht_entrylk (call_frame_t *frame, xlator_t *this,
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (loc, err);
         VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
 
         local = dht_local_init (frame, loc, NULL, GF_FOP_ENTRYLK);
         if (!local) {
@@ -4605,16 +5070,21 @@ err:
 int
 dht_forget (xlator_t *this, inode_t *inode)
 {
-        uint64_t      tmp_layout = 0;
+        uint64_t        ctx_int = 0;
+        dht_inode_ctx_t *ctx    = NULL;
         dht_layout_t *layout = NULL;
 
-        inode_ctx_del (inode, this, &tmp_layout);
+        inode_ctx_del (inode, this, &ctx_int);
 
-        if (!tmp_layout)
+        if (!ctx_int)
                 return 0;
 
-        layout = (dht_layout_t *)(long)tmp_layout;
+        ctx = (dht_inode_ctx_t *) (long) ctx_int;
+
+        layout = ctx->layout;
+        ctx->layout = NULL;
         dht_layout_unref (this, layout);
+        GF_FREE (ctx);
 
         return 0;
 }
@@ -4701,7 +5171,8 @@ dht_notify (xlator_t *this, int event, void *data, ...)
                         gf_log (this->name, GF_LOG_WARNING,
                                 "Received CHILD_DOWN. Exiting");
                         if (conf->defrag) {
-                                gf_defrag_stop (conf->defrag, NULL);
+                                gf_defrag_stop (conf->defrag,
+                                                GF_DEFRAG_STATUS_FAILED, NULL);
                         } else {
                                 kill (getpid(), SIGTERM);
                         }
@@ -4777,7 +5248,8 @@ dht_notify (xlator_t *this, int event, void *data, ...)
                         if (cmd == GF_DEFRAG_CMD_STATUS)
                                 gf_defrag_status_get (defrag, output);
                         else if (cmd == GF_DEFRAG_CMD_STOP)
-                                gf_defrag_stop (defrag, output);
+                                gf_defrag_stop (defrag,
+                                                GF_DEFRAG_STATUS_STOPPED, output);
                 }
 unlock:
                 UNLOCK (&defrag->lock);
@@ -4802,15 +5274,7 @@ unlock:
            or wait for anything else. Just propagate blindly */
         if (have_heard_from_all) {
                 propagate = 1;
-                if (conf->defrag) {
-                        ret = pthread_create (&conf->defrag->th, NULL,
-                                              gf_defrag_start, this);
-                        if (ret) {
-                                conf->defrag = NULL;
-                                GF_FREE (conf->defrag);
-                                kill (getpid(), SIGTERM);
-                        }
-                }
+
         }
 
 
@@ -4832,6 +5296,19 @@ unlock:
                                 /* continue to check other events for CHILD_UP */
                         }
                 }
+
+                /* rebalance is started with assert_no_child_down. So we do
+                 * not need to handle CHILD_DOWN event here.
+                 */
+                if (conf->defrag) {
+                        ret = gf_thread_create (&conf->defrag->th, NULL,
+						gf_defrag_start, this);
+                        if (ret) {
+                                conf->defrag = NULL;
+                                GF_FREE (conf->defrag);
+                                kill (getpid(), SIGTERM);
+                        }
+                }
         }
 
         ret = 0;
@@ -4840,3 +5317,24 @@ unlock:
 
         return ret;
 }
+
+int
+dht_inode_ctx_layout_get (inode_t *inode, xlator_t *this, dht_layout_t **layout)
+{
+        dht_inode_ctx_t         *ctx            = NULL;
+        int                      ret            = -1;
+
+        ret = dht_inode_ctx_get (inode, this, &ctx);
+
+        if (!ret && ctx) {
+                if (ctx->layout) {
+                        if (layout)
+                                *layout = ctx->layout;
+                        ret = 0;
+                } else {
+                        ret = -1;
+                }
+        }
+
+        return ret;
+}
diff --git a/xlators/cluster/dht/src/dht-common.h b/xlators/cluster/dht/src/dht-common.h
index 0af21d10d..d391b87d5 100644
--- a/xlators/cluster/dht/src/dht-common.h
+++ b/xlators/cluster/dht/src/dht-common.h
@@ -13,6 +13,9 @@
 #include "config.h"
 #endif
 
+#include <regex.h>
+#include <signal.h>
+
 #include "dht-mem-types.h"
 #include "libxlator.h"
 #include "syncop.h"
@@ -53,13 +56,32 @@ struct dht_layout {
                 uint32_t   start;
                 uint32_t   stop;
                 xlator_t  *xlator;
-        } list[0];
+        } list[];
 };
 typedef struct dht_layout  dht_layout_t;
 
+struct dht_stat_time {
+        uint32_t        atime;
+        uint32_t        atime_nsec;
+        uint32_t        ctime;
+        uint32_t        ctime_nsec;
+        uint32_t        mtime;
+        uint32_t        mtime_nsec;
+};
+
+typedef struct dht_stat_time dht_stat_time_t;
+
+struct dht_inode_ctx {
+        dht_layout_t    *layout;
+        dht_stat_time_t  time;
+};
+
+typedef struct dht_inode_ctx dht_inode_ctx_t;
+
 
 typedef enum {
         DHT_HASH_TYPE_DM,
+        DHT_HASH_TYPE_DM_USER,
 } dht_hashfn_type_t;
 
 /* rebalance related */
@@ -121,7 +143,6 @@ struct dht_local {
         struct {
                 uint32_t         hole_cnt;
                 uint32_t         overlaps_cnt;
-                uint32_t         missing;
                 uint32_t         down;
                 uint32_t         misc;
                 dht_selfheal_dir_cbk_t   dir_cbk;
@@ -142,6 +163,7 @@ struct dht_local {
 
         /* which xattr request? */
         char xsel[256];
+        int32_t alloc_len;
 
         char   *newpath;
 
@@ -162,6 +184,7 @@ struct dht_local {
         xlator_t        *link_subvol;
 
         struct dht_rebalance_ rebalance;
+        xlator_t        *first_up_subvol;
 
 };
 typedef struct dht_local dht_local_t;
@@ -190,15 +213,27 @@ enum gf_defrag_status_t {
         GF_DEFRAG_STATUS_STOPPED,
         GF_DEFRAG_STATUS_COMPLETE,
         GF_DEFRAG_STATUS_FAILED,
+        GF_DEFRAG_STATUS_LAYOUT_FIX_STARTED,
+        GF_DEFRAG_STATUS_LAYOUT_FIX_STOPPED,
+        GF_DEFRAG_STATUS_LAYOUT_FIX_COMPLETE,
+        GF_DEFRAG_STATUS_LAYOUT_FIX_FAILED,
 };
 typedef enum gf_defrag_status_t gf_defrag_status_t;
 
+typedef struct gf_defrag_pattern_list gf_defrag_pattern_list_t;
+
+struct gf_defrag_pattern_list {
+        char                       path_pattern[256];
+        uint64_t                   size;
+        gf_defrag_pattern_list_t  *next;
+};
 
 struct gf_defrag_info_ {
         uint64_t                     total_files;
         uint64_t                     total_data;
         uint64_t                     num_files_lookedup;
         uint64_t                     total_failures;
+        uint64_t                     skipped;
         gf_lock_t                    lock;
         int                          cmd;
         pthread_t                    th;
@@ -209,7 +244,9 @@ struct gf_defrag_info_ {
         pid_t                        pid;
         inode_t                     *root_inode;
         uuid_t                       node_uuid;
-
+        struct timeval               start_time;
+        gf_boolean_t                 stats;
+        gf_defrag_pattern_list_t    *defrag_pattern;
 };
 
 typedef struct gf_defrag_info_ gf_defrag_info_t;
@@ -225,8 +262,8 @@ struct dht_conf {
         gf_boolean_t   search_unhashed;
         int            gen;
         dht_du_t      *du_stats;
-        uint64_t       min_free_disk;
-	uint32_t       min_free_inodes;
+        double         min_free_disk;
+	double         min_free_inodes;
         char           disk_unit;
         int32_t        refresh_interval;
         gf_boolean_t   unhashed_sticky_bit;
@@ -246,6 +283,7 @@ struct dht_conf {
         /* to keep track of nodes which are decomissioned */
         xlator_t     **decommissioned_bricks;
         int            decommission_in_progress;
+        int            decommission_subvols_cnt;
 
         /* defrag related */
         gf_defrag_info_t *defrag;
@@ -253,6 +291,17 @@ struct dht_conf {
         /* Request to filter directory entries in readdir request */
 
         gf_boolean_t    readdir_optimize;
+
+        /* Support regex-based name reinterpretation. */
+        regex_t         rsync_regex;
+        gf_boolean_t    rsync_regex_valid;
+        regex_t         extra_regex;
+        gf_boolean_t    extra_regex_valid;
+
+        /* Support variable xattr names. */
+        char            *xattr_name;
+        char            *link_xattr_name;
+        char            *wild_xattr_name;
 };
 typedef struct dht_conf dht_conf_t;
 
@@ -276,20 +325,14 @@ typedef enum {
 
 #define ENTRY_MISSING(op_ret, op_errno) (op_ret == -1 && op_errno == ENOENT)
 
-#define is_revalidate(loc) (inode_ctx_get (loc->inode, this, NULL) == 0)
+#define is_revalidate(loc) (dht_inode_ctx_layout_get (loc->inode, this, NULL) == 0)
 
 #define is_last_call(cnt) (cnt == 0)
 
 #define DHT_MIGRATION_IN_PROGRESS 1
 #define DHT_MIGRATION_COMPLETED   2
 
-#define DHT_LINKFILE_KEY         "trusted.glusterfs.dht.linkto"
-#define DHT_LINKFILE_MODE        (S_ISVTX)
-
-#define check_is_linkfile(i,s,x) (                                      \
-                ((st_mode_from_ia ((s)->ia_prot, (s)->ia_type) & ~S_IFMT) \
-                 == DHT_LINKFILE_MODE) &&                               \
-                dict_get (x, DHT_LINKFILE_KEY))
+#define check_is_linkfile(i,s,x,n) (IS_DHT_LINKFILE_MODE (s) && dict_get (x, n))
 
 #define IS_DHT_MIGRATION_PHASE2(buf)  (                                 \
                 IA_ISREG ((buf)->ia_type) &&                            \
@@ -308,6 +351,8 @@ typedef enum {
                 }                                               \
         } while (0)
 
+#define dht_inode_missing(op_errno) (op_errno == ENOENT || op_errno == ESTALE)
+
 #define check_is_dir(i,s,x) (IA_ISDIR(s->ia_type))
 
 #define layout_is_sane(layout) ((layout) && (layout->cnt > 0))
@@ -334,6 +379,25 @@ typedef enum {
                 dht_local_wipe (__xl, __local);         \
         } while (0)
 
+#define DHT_UPDATE_TIME(ctx_sec, ctx_nsec, new_sec, new_nsec, inode, post) do {\
+                int32_t sec = 0;                                        \
+                sec = new_sec;                                          \
+                LOCK (&inode->lock);                                    \
+                {                                                       \
+                        new_sec = max(new_sec, ctx_sec);                \
+                        if (sec < new_sec)                              \
+                                new_nsec = ctx_nsec;                    \
+                        if (sec == new_sec)                             \
+                                new_nsec = max (new_nsec, ctx_nsec);    \
+                        if (post) {                                     \
+                                ctx_sec = new_sec;                      \
+                                ctx_nsec = new_nsec;                    \
+                        }                                               \
+                }                                                       \
+                UNLOCK (&inode->lock);                                  \
+        } while (0)
+
+#define is_greater_time(a, an, b, bn) (((a) < (b)) || (((a) == (b)) && ((an) < (bn))))
 dht_layout_t                            *dht_layout_new (xlator_t *this, int cnt);
 dht_layout_t                            *dht_layout_get (xlator_t *this, inode_t *inode);
 dht_layout_t                            *dht_layout_for_subvol (xlator_t *this, xlator_t *subvol);
@@ -343,7 +407,7 @@ int                                      dht_layout_normalize (xlator_t *this, l
 int dht_layout_anomalies (xlator_t      *this, loc_t *loc, dht_layout_t *layout,
                           uint32_t      *holes_p, uint32_t *overlaps_p,
                           uint32_t      *missing_p, uint32_t *down_p,
-                          uint32_t      *misc_p);
+                          uint32_t      *misc_p, uint32_t *no_space_p);
 int dht_layout_dir_mismatch (xlator_t   *this, dht_layout_t *layout,
                              xlator_t   *subvol, loc_t *loc, dict_t *xattr);
 
@@ -377,12 +441,14 @@ int dht_iatt_merge (xlator_t                 *this, struct iatt *to, struct iatt
 xlator_t *dht_subvol_get_hashed (xlator_t *this, loc_t *loc);
 xlator_t *dht_subvol_get_cached (xlator_t *this, inode_t *inode);
 xlator_t *dht_subvol_next (xlator_t *this, xlator_t *prev);
+xlator_t *dht_subvol_next_available (xlator_t *this, xlator_t *prev);
 int       dht_subvol_cnt (xlator_t *this, xlator_t *subvol);
 
-int dht_hash_compute (int type, const char *name, uint32_t *hash_p);
+int dht_hash_compute (xlator_t *this, int type, const char *name, uint32_t *hash_p);
 
 int dht_linkfile_create (call_frame_t    *frame, fop_mknod_cbk_t linkfile_cbk,
-                         xlator_t        *tovol, xlator_t *fromvol, loc_t *loc);
+                         xlator_t        *this, xlator_t *tovol,
+                         xlator_t        *fromvol, loc_t *loc);
 int                                       dht_lookup_directory (call_frame_t *frame, xlator_t *this, loc_t *loc);
 int                                       dht_lookup_everywhere (call_frame_t *frame, xlator_t *this, loc_t *loc);
 int
@@ -400,7 +466,8 @@ dht_layout_sort_volname (dht_layout_t *layout);
 int dht_get_du_info (call_frame_t *frame, xlator_t *this, loc_t *loc);
 
 gf_boolean_t dht_is_subvol_filled (xlator_t *this, xlator_t *subvol);
-xlator_t *dht_free_disk_available_subvol (xlator_t *this, xlator_t *subvol);
+xlator_t *dht_free_disk_available_subvol (xlator_t *this, xlator_t *subvol,
+                                          dht_local_t *layout);
 int       dht_get_du_info_for_subvol (xlator_t *this, int subvol_idx);
 
 int dht_layout_preset (xlator_t *this, xlator_t *subvol, inode_t *inode);
@@ -622,7 +689,16 @@ int32_t dht_setattr (call_frame_t  *frame, xlator_t *this, loc_t *loc,
                      struct iatt   *stbuf, int32_t valid, dict_t *xdata);
 int32_t dht_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
                       struct iatt  *stbuf, int32_t valid, dict_t *xdata);
-
+int32_t dht_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd,
+		      int32_t mode, off_t offset, size_t len, dict_t *xdata);
+int32_t dht_discard(call_frame_t *frame, xlator_t *this, fd_t *fd,
+		    off_t offset, size_t len, dict_t *xdata);
+int32_t dht_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                    off_t offset, off_t len, dict_t *xdata);
+
+int32_t dht_init (xlator_t *this);
+void    dht_fini (xlator_t *this);
+int     dht_reconfigure (xlator_t *this, dict_t *options);
 int32_t dht_notify (xlator_t *this, int32_t event, void *data, ...);
 
 /* definitions for nufa/switch */
@@ -656,7 +732,8 @@ int
 gf_defrag_status_get (gf_defrag_info_t *defrag, dict_t *dict);
 
 int
-gf_defrag_stop (gf_defrag_info_t *defrag, dict_t *output);
+gf_defrag_stop (gf_defrag_info_t *defrag, gf_defrag_status_t status,
+                dict_t *output);
 
 void*
 gf_defrag_start (void *this);
@@ -667,11 +744,43 @@ gf_defrag_handle_hardlink (xlator_t *this, loc_t *loc, dict_t  *xattrs,
 int
 dht_migrate_file (xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
                  int flag);
-xlator_t *
-dht_subvol_with_free_space_inodes (xlator_t *this, xlator_t *subvol);
+int
+dht_inode_ctx_layout_get (inode_t *inode, xlator_t *this,
+                          dht_layout_t **layout_int);
+int
+dht_inode_ctx_layout_set (inode_t *inode, xlator_t *this,
+                          dht_layout_t* layout_int);
+int
+dht_inode_ctx_time_update (inode_t *inode, xlator_t *this, struct iatt *stat,
+                           int32_t update_ctx);
 
+int dht_inode_ctx_get (inode_t *inode, xlator_t *this, dht_inode_ctx_t **ctx);
+int dht_inode_ctx_set (inode_t *inode, xlator_t *this, dht_inode_ctx_t *ctx);
+int
+dht_dir_attr_heal (void *data);
+int
+dht_dir_attr_heal_done (int ret, call_frame_t *sync_frame, void *data);
+int
+dht_dir_has_layout (dict_t *xattr, char *name);
+gf_boolean_t
+dht_is_subvol_in_layout (dht_layout_t *layout, xlator_t *xlator);
 xlator_t *
-dht_subvol_maxspace_nonzeroinode (xlator_t *this, xlator_t *subvol);
+dht_subvol_with_free_space_inodes (xlator_t *this, xlator_t *subvol,
+                                   dht_layout_t *layout);
+xlator_t *
+dht_subvol_maxspace_nonzeroinode (xlator_t *this, xlator_t *subvol,
+                                  dht_layout_t *layout);
 int
 dht_linkfile_attr_heal (call_frame_t *frame, xlator_t *this);
+
+void
+dht_layout_dump (dht_layout_t  *layout, const char *prefix);
+int32_t
+dht_priv_dump (xlator_t *this);
+int32_t
+dht_inodectx_dump (xlator_t *this, inode_t *inode);
+
+int
+dht_inode_ctx_get1 (xlator_t *this, inode_t *inode, xlator_t **subvol);
+
 #endif/* _DHT_H */
diff --git a/xlators/cluster/dht/src/dht-diskusage.c b/xlators/cluster/dht/src/dht-diskusage.c
index 0c87f4a64..fe3955ecb 100644
--- a/xlators/cluster/dht/src/dht-diskusage.c
+++ b/xlators/cluster/dht/src/dht-diskusage.c
@@ -251,25 +251,45 @@ dht_is_subvol_filled (xlator_t *this, xlator_t *subvol)
 
 /*Get the best subvolume to create the file in*/
 xlator_t *
-dht_free_disk_available_subvol (xlator_t *this, xlator_t *subvol)
+dht_free_disk_available_subvol (xlator_t *this, xlator_t *subvol,
+                                dht_local_t *local)
 {
 	xlator_t   *avail_subvol = NULL;
 	dht_conf_t *conf = NULL;
+        dht_layout_t *layout = NULL;
+        loc_t      *loc = NULL;
 
 	conf = this->private;
+        if (!local)
+                goto out;
+        loc = &local->loc;
+        if (!local->layout) {
+                layout = dht_layout_get (this, loc->parent);
+
+                if (!layout) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "layout missing path=%s parent=%s",
+                                loc->path, uuid_utoa (loc->parent->gfid));
+                        goto out;
+                }
+        } else {
+                layout = dht_layout_ref (this, local->layout);
+        }
 
-	LOCK (&conf->subvolume_lock);
+        LOCK (&conf->subvolume_lock);
 	{
-                avail_subvol = dht_subvol_with_free_space_inodes(this, subvol);
+                avail_subvol = dht_subvol_with_free_space_inodes(this, subvol,
+                                                                 layout);
                 if(!avail_subvol)
                 {
                         avail_subvol = dht_subvol_maxspace_nonzeroinode(this,
-                                                                        subvol);
+                                                                        subvol,
+                                                                        layout);
                 }
 
 	}
 	UNLOCK (&conf->subvolume_lock);
-
+out:
 	if (!avail_subvol) {
 		gf_log (this->name,
                         GF_LOG_DEBUG,
@@ -278,17 +298,42 @@ dht_free_disk_available_subvol (xlator_t *this, xlator_t *subvol)
                 avail_subvol = subvol;
 	}
 
-
+        if (layout)
+                dht_layout_unref (this, layout);
 	return avail_subvol;
 }
 
+static inline
+int32_t dht_subvol_has_err (xlator_t *this, dht_layout_t *layout)
+{
+        int ret = -1;
+        int i   = 0;
+
+        if (!this || !layout)
+                goto out;
+
+        /* check if subvol has layout errors, before selecting it */
+        for (i = 0; i < layout->cnt; i++) {
+                if (!strcmp (layout->list[i].xlator->name, this->name) &&
+                     (layout->list[i].err != 0)) {
+                        ret = -1;
+                        goto out;
+                }
+        }
+        ret = 0;
+out:
+        return ret;
+}
+
 /*Get subvolume which has both space and inodes more than the min criteria*/
 xlator_t *
-dht_subvol_with_free_space_inodes(xlator_t *this, xlator_t *subvol)
+dht_subvol_with_free_space_inodes(xlator_t *this, xlator_t *subvol,
+                                  dht_layout_t *layout)
 {
         int i = 0;
         double max = 0;
         double max_inodes = 0;
+        int    ignore_subvol = 0;
 
         xlator_t *avail_subvol = NULL;
         dht_conf_t *conf = NULL;
@@ -296,6 +341,12 @@ dht_subvol_with_free_space_inodes(xlator_t *this, xlator_t *subvol)
         conf = this->private;
 
         for(i=0; i < conf->subvolume_cnt; i++) {
+                /* check if subvol has layout errors, before selecting it */
+                ignore_subvol = dht_subvol_has_err (conf->subvolumes[i],
+                                                    layout);
+                if (ignore_subvol)
+                        continue;
+
                 if ((conf->disk_unit == 'p') &&
                     (conf->du_stats[i].avail_percent > conf->min_free_disk) &&
                     (conf->du_stats[i].avail_inodes  > conf->min_free_inodes)) {
@@ -325,10 +376,12 @@ dht_subvol_with_free_space_inodes(xlator_t *this, xlator_t *subvol)
 
 /* Get subvol which has atleast one inode and maximum space */
 xlator_t *
-dht_subvol_maxspace_nonzeroinode (xlator_t *this, xlator_t *subvol)
+dht_subvol_maxspace_nonzeroinode (xlator_t *this, xlator_t *subvol,
+                                  dht_layout_t *layout)
 {
         int         i = 0;
         double      max = 0;
+        int         ignore_subvol = 0;
 
         xlator_t   *avail_subvol = NULL;
         dht_conf_t *conf = NULL;
@@ -336,6 +389,12 @@ dht_subvol_maxspace_nonzeroinode (xlator_t *this, xlator_t *subvol)
         conf = this->private;
 
         for (i = 0; i < conf->subvolume_cnt; i++) {
+                /* check if subvol has layout errors, before selecting it */
+                ignore_subvol = dht_subvol_has_err (conf->subvolumes[i],
+                                                    layout);
+                if (ignore_subvol)
+                        continue;
+
                 if (conf->disk_unit == 'p') {
                         if ((conf->du_stats[i].avail_percent > max)
                             && (conf->du_stats[i].avail_inodes > 0 )) {
diff --git a/xlators/cluster/dht/src/dht-hashfn.c b/xlators/cluster/dht/src/dht-hashfn.c
index 98756157d..656cf23a0 100644
--- a/xlators/cluster/dht/src/dht-hashfn.c
+++ b/xlators/cluster/dht/src/dht-hashfn.c
@@ -28,6 +28,7 @@ dht_hash_compute_internal (int type, const char *name, uint32_t *hash_p)
 
         switch (type) {
         case DHT_HASH_TYPE_DM:
+        case DHT_HASH_TYPE_DM_USER:
                 hash = gf_dm_hashfn (name, strlen (name));
                 break;
         default:
@@ -43,30 +44,68 @@ dht_hash_compute_internal (int type, const char *name, uint32_t *hash_p)
 }
 
 
-#define MAKE_RSYNC_FRIENDLY_NAME(rsync_frndly_name, name) do {          \
-                rsync_frndly_name = (char *) name;                      \
-                if (name[0] == '.') {                                   \
-                        char *dot   = 0;                                \
-                        int namelen = 0;                                \
-                                                                        \
-                        dot = strrchr (name, '.');                      \
-                        if (dot && dot > (name + 1) && *(dot + 1)) {    \
-                                namelen = (dot - name);                 \
-                                rsync_frndly_name = alloca (namelen);   \
-                                strncpy (rsync_frndly_name, name + 1,   \
-                                         namelen);                      \
-                                rsync_frndly_name[namelen - 1] = 0;     \
-                        }                                               \
-                }                                                       \
-        } while (0);
+static inline
+gf_boolean_t
+dht_munge_name (const char *original, char *modified, size_t len, regex_t *re)
+{
+        regmatch_t      matches[2];
+        size_t          new_len;
 
+        if (regexec(re,original,2,matches,0) != REG_NOMATCH) {
+                if (matches[1].rm_so != -1) {
+                        new_len = matches[1].rm_eo - matches[1].rm_so;
+                        /* Equal would fail due to the NUL at the end. */
+                        if (new_len < len) {
+                                memcpy (modified,original+matches[1].rm_so,
+                                        new_len);
+                                modified[new_len] = '\0';
+                                return _gf_true;
+                        }
+                }
+        }
+
+        /* This is guaranteed safe because of how the dest was allocated. */
+        strcpy(modified,original);
+        return _gf_false;
+}
 
 int
-dht_hash_compute (int type, const char *name, uint32_t *hash_p)
+dht_hash_compute (xlator_t *this, int type, const char *name, uint32_t *hash_p)
 {
-        char     *rsync_friendly_name = NULL;
+        char            *rsync_friendly_name    = NULL;
+        dht_conf_t      *priv                   = this->private;
+        size_t           len                    = 0;
+        gf_boolean_t     munged                 = _gf_false;
+
+        /*
+         * It wouldn't be safe to use alloca in an inline function that doesn't
+         * actually get inlined, and it wouldn't be efficient to do a real
+         * allocation, so we use alloca here (if needed) and pass that to the
+         * inline.
+         */
 
-        MAKE_RSYNC_FRIENDLY_NAME (rsync_friendly_name, name);
+        if (priv->extra_regex_valid) {
+                len = strlen(name) + 1;
+                rsync_friendly_name = alloca(len);
+                munged = dht_munge_name (name, rsync_friendly_name, len,
+                                         &priv->extra_regex);
+        }
+
+        if (!munged && priv->rsync_regex_valid) {
+                len = strlen(name) + 1;
+                rsync_friendly_name = alloca(len);
+                gf_log (this->name, GF_LOG_TRACE, "trying regex for %s", name);
+                munged = dht_munge_name (name, rsync_friendly_name, len,
+                                         &priv->rsync_regex);
+                if (munged) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "munged down to %s", rsync_friendly_name);
+                }
+        }
+
+        if (!munged) {
+                rsync_friendly_name = (char *)name;
+        }
 
         return dht_hash_compute_internal (type, rsync_friendly_name, hash_p);
 }
diff --git a/xlators/cluster/dht/src/dht-helper.c b/xlators/cluster/dht/src/dht-helper.c
index 18a0d9de2..18a501f04 100644
--- a/xlators/cluster/dht/src/dht-helper.c
+++ b/xlators/cluster/dht/src/dht-helper.c
@@ -18,6 +18,28 @@
 #include "xlator.h"
 #include "dht-common.h"
 
+static inline int
+dht_inode_ctx_set1 (xlator_t *this, inode_t *inode, xlator_t *subvol)
+{
+        uint64_t tmp_subvol = 0;
+
+        tmp_subvol = (long)subvol;
+        return inode_ctx_set1 (inode, this, &tmp_subvol);
+}
+
+int
+dht_inode_ctx_get1 (xlator_t *this, inode_t *inode, xlator_t **subvol)
+{
+        int ret = -1;
+        uint64_t tmp_subvol = 0;
+
+        ret =  inode_ctx_get1 (inode, this, &tmp_subvol);
+        if (tmp_subvol && subvol)
+                *subvol = (xlator_t *)tmp_subvol;
+
+        return ret;
+}
+
 
 int
 dht_frame_return (call_frame_t *frame)
@@ -136,7 +158,7 @@ dht_filter_loc_subvol_key (xlator_t *this, loc_t *loc, loc_t *new_loc,
         int            ret       = 0; /* not found */
 
         /* Why do other tasks if first required 'char' itself is not there */
-        if (!loc->name || !strchr (loc->name, '@'))
+        if (!new_loc || !loc || !loc->name || !strchr (loc->name, '@'))
                 goto out;
 
         trav = this->children;
@@ -176,10 +198,8 @@ dht_filter_loc_subvol_key (xlator_t *this, loc_t *loc, loc_t *new_loc,
 out:
         if (!ret) {
                 /* !success */
-                if (new_path)
-                        GF_FREE (new_path);
-                if (new_name)
-                        GF_FREE (new_name);
+                GF_FREE (new_path);
+                GF_FREE (new_name);
         }
         return ret;
 }
@@ -283,16 +303,11 @@ dht_local_wipe (xlator_t *this, dht_local_t *local)
                 local->selfheal.layout = NULL;
         }
 
-        if (local->newpath) {
-                GF_FREE (local->newpath);
-        }
+        GF_FREE (local->newpath);
 
-        if (local->key) {
-                GF_FREE (local->key);
-        }
+        GF_FREE (local->key);
 
-        if (local->rebalance.vector)
-                GF_FREE (local->rebalance.vector);
+        GF_FREE (local->rebalance.vector);
 
         if (local->rebalance.iobref)
                 iobref_unref (local->rebalance.iobref);
@@ -347,20 +362,6 @@ out:
         return local;
 }
 
-
-char *
-basestr (const char *str)
-{
-        char *basestr = NULL;
-
-        basestr = strrchr (str, '/');
-        if (basestr)
-                basestr ++;
-
-        return basestr;
-}
-
-
 xlator_t *
 dht_first_up_subvol (xlator_t *this)
 {
@@ -512,7 +513,36 @@ out:
         return next;
 }
 
+/* This func wraps around, if prev is actually the last subvol.
+ */
+xlator_t *
+dht_subvol_next_available (xlator_t *this, xlator_t *prev)
+{
+        dht_conf_t *conf = NULL;
+        int         i = 0;
+        xlator_t   *next = NULL;
+
+        conf = this->private;
+        if (!conf)
+                goto out;
 
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (conf->subvolumes[i] == prev) {
+                        /* if prev is last in conf->subvolumes, then wrap
+                         * around.
+                         */
+                        if ((i + 1) < conf->subvolume_cnt) {
+                                next = conf->subvolumes[i + 1];
+                        } else {
+                                next = conf->subvolumes[0];
+                        }
+                        break;
+                }
+        }
+
+out:
+        return next;
+}
 int
 dht_subvol_cnt (xlator_t *this, xlator_t *subvol)
 {
@@ -704,39 +734,61 @@ dht_migration_complete_check_task (void *data)
         call_frame_t *frame    = NULL;
         loc_t         tmp_loc  = {0,};
         char         *path     = NULL;
+        dht_conf_t   *conf     = NULL;
+        inode_t      *inode    = NULL;
+        fd_t         *iter_fd  = NULL;
+        uint64_t      tmp_subvol = 0;
+        int           open_failed = 0;
 
         this  = THIS;
         frame = data;
         local = frame->local;
+        conf = this->private;
 
         src_node = local->cached_subvol;
 
-        if (!local->loc.inode && !local->fd)
+        if (!local->loc.inode && !local->fd) {
+                local->op_errno = EINVAL;
                 goto out;
+        }
+
+        inode = (!local->fd) ? local->loc.inode : local->fd->inode;
 
-        /* getxattr on cached_subvol for 'linkto' value */
-        if (!local->loc.inode)
+        /* getxattr on cached_subvol for 'linkto' value. Do path based getxattr
+         * as root:root. If a fd is already open, access check wont be done*/
+
+        if (!local->loc.inode) {
                 ret = syncop_fgetxattr (src_node, local->fd, &dict,
-                                        DHT_LINKFILE_KEY);
-        else
+                                        conf->link_xattr_name);
+        } else {
+                SYNCTASK_SETID (0, 0);
                 ret = syncop_getxattr (src_node, &local->loc, &dict,
-                                       DHT_LINKFILE_KEY);
+                                       conf->link_xattr_name);
+                SYNCTASK_SETID (frame->root->uid, frame->root->gid);
+        }
 
         if (!ret)
                 dst_node = dht_linkfile_subvol (this, NULL, NULL, dict);
 
         if (ret) {
-                if ((errno != ENOENT) || (!local->loc.inode)) {
+                if (!dht_inode_missing(-ret) || (!local->loc.inode)) {
+                        local->op_errno = -ret;
                         gf_log (this->name, GF_LOG_ERROR,
                                 "%s: failed to get the 'linkto' xattr %s",
-                                local->loc.path, strerror (errno));
+                                local->loc.path, strerror (-ret));
+                        ret = -1;
                         goto out;
                 }
+
                 /* Need to do lookup on hashed subvol, then get the file */
                 ret = syncop_lookup (this, &local->loc, NULL, &stbuf, NULL,
                                      NULL);
-                if (ret)
+                if (ret) {
+                        local->op_errno = -ret;
+                        ret = -1;
                         goto out;
+                }
+
                 dst_node = dht_subvol_get_cached (this, local->loc.inode);
         }
 
@@ -745,17 +797,21 @@ dht_migration_complete_check_task (void *data)
                         "%s: failed to get the destination node",
                         local->loc.path);
                 ret = -1;
+                local->op_errno = EINVAL;
                 goto out;
         }
 
         /* lookup on dst */
         if (local->loc.inode) {
-                ret = syncop_lookup (dst_node, &local->loc, NULL, &stbuf, NULL, NULL);
+                ret = syncop_lookup (dst_node, &local->loc, NULL, &stbuf, NULL,
+                                     NULL);
 
                 if (ret) {
                         gf_log (this->name, GF_LOG_ERROR,
                                 "%s: failed to lookup the file on %s",
                                 local->loc.path, dst_node->name);
+                        local->op_errno = -ret;
+                        ret = -1;
                         goto out;
                 }
 
@@ -764,6 +820,7 @@ dht_migration_complete_check_task (void *data)
                                 "%s: gfid different on the target file on %s",
                                 local->loc.path, dst_node->name);
                         ret = -1;
+                        local->op_errno = EIO;
                         goto out;
                 }
         }
@@ -771,15 +828,13 @@ dht_migration_complete_check_task (void *data)
         /* update inode ctx (the layout) */
         dht_layout_unref (this, local->layout);
 
-        if (!local->loc.inode)
-                ret = dht_layout_preset (this, dst_node, local->fd->inode);
-        else
-                ret = dht_layout_preset (this, dst_node, local->loc.inode);
+        ret = dht_layout_preset (this, dst_node, inode);
         if (ret != 0) {
                 gf_log (this->name, GF_LOG_DEBUG,
                         "%s: could not set preset layout for subvol %s",
                         local->loc.path, dst_node->name);
                 ret   = -1;
+                local->op_errno = EINVAL;
                 goto out;
         }
 
@@ -789,55 +844,69 @@ dht_migration_complete_check_task (void *data)
                         "%s: no pre-set layout for subvolume %s",
                         local->loc.path, dst_node ? dst_node->name : "<nil>");
                 ret = -1;
+                local->op_errno = EINVAL;
                 goto out;
         }
 
-        if (!local->loc.inode)
-                ret = dht_layout_set (this, local->fd->inode, layout);
-        else
-                ret = dht_layout_set (this, local->loc.inode, layout);
+        ret = dht_layout_set (this, inode, layout);
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "%s: failed to set the new layout",
                         local->loc.path);
+                local->op_errno = EINVAL;
                 goto out;
         }
 
         local->cached_subvol = dst_node;
         ret = 0;
 
-        /* once we detect the migration complete, the fd-ctx is no more
-           required.. delete the ctx, and do one extra 'fd_unref' for open fd */
-        ret = fd_ctx_del (local->fd, this, NULL);
-        if (!ret) {
-                fd_unref (local->fd);
-                ret = 0;
+        /* once we detect the migration complete, the inode-ctx2 is no more
+           required.. delete the ctx and also, it means, open() already
+           done on all the fd of inode */
+        ret = inode_ctx_reset1 (inode, this, &tmp_subvol);
+        if (tmp_subvol)
                 goto out;
-        }
 
-        /* if 'local->fd' (ie, fd based operation), send a 'open()' on
-           destination if not already done */
-        if (local->loc.inode) {
-                ret = syncop_open (dst_node, &local->loc,
-                                   local->fd->flags, local->fd);
-        } else {
-                tmp_loc.inode = local->fd->inode;
-                inode_path (local->fd->inode, NULL, &path);
-                if (path)
-                        tmp_loc.path = path;
-                ret = syncop_open (dst_node, &tmp_loc,
-                                   local->fd->flags, local->fd);
-                if (path)
-                        GF_FREE (path);
+        if (list_empty (&inode->fd_list))
+                goto out;
 
+        /* perform open as root:root. There is window between linkfile
+         * creation(root:root) and setattr with the correct uid/gid
+         */
+        SYNCTASK_SETID(0, 0);
+
+        /* perform 'open()' on all the fd's present on the inode */
+        tmp_loc.inode = inode;
+        inode_path (inode, NULL, &path);
+        if (path)
+                tmp_loc.path = path;
+        list_for_each_entry (iter_fd, &inode->fd_list, inode_list) {
+                if (fd_is_anonymous (iter_fd))
+                        continue;
+
+                /* flags for open are stripped down to allow following the
+                 * new location of the file, otherwise we can get EEXIST or
+                 * truncate the file again as rebalance is moving the data */
+                ret = syncop_open (dst_node, &tmp_loc,
+                                   (iter_fd->flags &
+                                   ~(O_CREAT | O_EXCL | O_TRUNC)), iter_fd);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR, "failed to open "
+                                "the fd (%p, flags=0%o) on file %s @ %s",
+                                iter_fd, iter_fd->flags, path, dst_node->name);
+                        open_failed = 1;
+                        local->op_errno = -ret;
+                        ret = -1;
+                }
         }
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s: failed to send open() on target file at %s",
-                        local->loc.path, dst_node->name);
+        GF_FREE (path);
+
+        SYNCTASK_SETID (frame->root->uid, frame->root->gid);
+
+        if (open_failed) {
+                ret = -1;
                 goto out;
         }
-
         ret = 0;
 out:
 
@@ -881,29 +950,41 @@ dht_rebalance_inprogress_task (void *data)
         char         *path     = NULL;
         struct iatt   stbuf    = {0,};
         loc_t         tmp_loc  = {0,};
+        dht_conf_t   *conf     = NULL;
+        inode_t      *inode    = NULL;
+        fd_t         *iter_fd  = NULL;
+        int           open_failed = 0;
 
         this  = THIS;
         frame = data;
         local = frame->local;
+        conf = this->private;
 
         src_node = local->cached_subvol;
 
         if (!local->loc.inode && !local->fd)
                 goto out;
 
-        /* getxattr on cached_subvol for 'linkto' value */
-        if (local->loc.inode)
+        inode = (!local->fd) ? local->loc.inode : local->fd->inode;
+
+        /* getxattr on cached_subvol for 'linkto' value. Do path based getxattr
+         * as root:root. If a fd is already open, access check wont be done*/
+        if (local->loc.inode) {
+                SYNCTASK_SETID (0, 0);
                 ret = syncop_getxattr (src_node, &local->loc, &dict,
-                                       DHT_LINKFILE_KEY);
-        else
+                                       conf->link_xattr_name);
+                SYNCTASK_SETID (frame->root->uid, frame->root->gid);
+        } else {
                 ret = syncop_fgetxattr (src_node, local->fd, &dict,
-                                        DHT_LINKFILE_KEY);
+                                        conf->link_xattr_name);
+        }
 
-        if (ret) {
+        if (ret < 0) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "%s: failed to get the 'linkto' xattr %s",
-                        local->loc.path, strerror (errno));
-                        goto out;
+                        local->loc.path, strerror (-ret));
+                ret = -1;
+                goto out;
         }
 
         dst_node = dht_linkfile_subvol (this, NULL, NULL, dict);
@@ -925,6 +1006,7 @@ dht_rebalance_inprogress_task (void *data)
                         gf_log (this->name, GF_LOG_ERROR,
                                 "%s: failed to lookup the file on %s",
                                 local->loc.path, dst_node->name);
+                        ret = -1;
                         goto out;
                 }
 
@@ -939,31 +1021,51 @@ dht_rebalance_inprogress_task (void *data)
 
         ret = 0;
 
-        if (local->loc.inode) {
-                ret = syncop_open (dst_node, &local->loc,
-                                   local->fd->flags, local->fd);
-        } else {
-                tmp_loc.inode = local->fd->inode;
-                inode_path (local->fd->inode, NULL, &path);
-                if (path)
-                        tmp_loc.path = path;
+        if (list_empty (&inode->fd_list))
+                goto done;
+
+        /* perform open as root:root. There is window between linkfile
+         * creation(root:root) and setattr with the correct uid/gid
+         */
+        SYNCTASK_SETID (0, 0);
+
+        tmp_loc.inode = inode;
+        inode_path (inode, NULL, &path);
+        if (path)
+                tmp_loc.path = path;
+
+        list_for_each_entry (iter_fd, &inode->fd_list, inode_list) {
+                if (fd_is_anonymous (iter_fd))
+                        continue;
+
+                /* flags for open are stripped down to allow following the
+                 * new location of the file, otherwise we can get EEXIST or
+                 * truncate the file again as rebalance is moving the data */
                 ret = syncop_open (dst_node, &tmp_loc,
-                                   local->fd->flags, local->fd);
-                if (path)
-                        GF_FREE (path);
+                                   (iter_fd->flags &
+                                   ~(O_CREAT | O_EXCL | O_TRUNC)), iter_fd);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR, "failed to send open "
+                                "the fd (%p, flags=0%o) on file %s @ %s",
+                                iter_fd, iter_fd->flags, path, dst_node->name);
+                        ret = -1;
+                        open_failed = 1;
+                }
         }
+        GF_FREE (path);
 
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s: failed to send open() on target file at %s",
-                        local->loc.path, dst_node->name);
+        SYNCTASK_SETID (frame->root->uid, frame->root->gid);
+
+        if (open_failed) {
+                ret = -1;
                 goto out;
         }
 
-        ret = fd_ctx_set (local->fd, this, (uint64_t)(long)dst_node);
+done:
+        ret = dht_inode_ctx_set1 (this, inode, dst_node);
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR,
-                        "%s: failed to set fd-ctx target file at %s",
+                        "%s: failed to set inode-ctx target file at %s",
                         local->loc.path, dst_node->name);
                 goto out;
         }
@@ -984,3 +1086,93 @@ dht_rebalance_in_progress_check (xlator_t *this, call_frame_t *frame)
                             frame, frame);
         return ret;
 }
+
+int
+dht_inode_ctx_layout_set (inode_t *inode, xlator_t *this,
+                          dht_layout_t *layout_int)
+{
+        dht_inode_ctx_t         *ctx            = NULL;
+        int                      ret            = -1;
+
+        ret = dht_inode_ctx_get (inode, this, &ctx);
+        if (!ret && ctx) {
+                ctx->layout = layout_int;
+        } else {
+                ctx = GF_CALLOC (1, sizeof (*ctx), gf_dht_mt_inode_ctx_t);
+                if (!ctx)
+                        return ret;
+                ctx->layout = layout_int;
+        }
+
+        ret = dht_inode_ctx_set (inode, this, ctx);
+
+        return ret;
+}
+
+int
+dht_inode_ctx_time_update (inode_t *inode, xlator_t *this, struct iatt *stat,
+                           int32_t post)
+{
+        dht_inode_ctx_t         *ctx            = NULL;
+        dht_stat_time_t         *time           = 0;
+        int                      ret            = -1;
+
+        GF_VALIDATE_OR_GOTO (this->name, stat, out);
+        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+
+        ret = dht_inode_ctx_get (inode, this, &ctx);
+
+        if (ret) {
+                ctx = GF_CALLOC (1, sizeof (*ctx), gf_dht_mt_inode_ctx_t);
+                if (!ctx)
+                        return -1;
+        }
+
+        time = &ctx->time;
+
+        DHT_UPDATE_TIME(time->mtime, time->mtime_nsec,
+                        stat->ia_mtime, stat->ia_mtime_nsec, inode, post);
+        DHT_UPDATE_TIME(time->ctime, time->ctime_nsec,
+                        stat->ia_ctime, stat->ia_ctime_nsec, inode, post);
+        DHT_UPDATE_TIME(time->atime, time->atime_nsec,
+                        stat->ia_atime, stat->ia_atime_nsec, inode, post);
+
+        ret = dht_inode_ctx_set (inode, this, ctx);
+out:
+        return 0;
+}
+
+int
+dht_inode_ctx_get (inode_t *inode, xlator_t *this, dht_inode_ctx_t **ctx)
+{
+        int             ret     = -1;
+        uint64_t        ctx_int = 0;
+
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+
+        ret = inode_ctx_get (inode, this, &ctx_int);
+
+        if (ret)
+                return ret;
+
+        if (ctx)
+                *ctx = (dht_inode_ctx_t *) ctx_int;
+out:
+        return ret;
+}
+
+int dht_inode_ctx_set (inode_t *inode, xlator_t *this, dht_inode_ctx_t *ctx)
+{
+        int             ret = -1;
+        uint64_t        ctx_int = 0;
+
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+        GF_VALIDATE_OR_GOTO (this->name, ctx, out);
+
+        ctx_int = (long)ctx;
+        ret = inode_ctx_set (inode, this, &ctx_int);
+out:
+        return ret;
+}
diff --git a/xlators/cluster/dht/src/dht-inode-read.c b/xlators/cluster/dht/src/dht-inode-read.c
index 7e29a7a8c..e8a9a7196 100644
--- a/xlators/cluster/dht/src/dht-inode-read.c
+++ b/xlators/cluster/dht/src/dht-inode-read.c
@@ -35,7 +35,7 @@ dht_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         prev = cookie;
 
         local->op_errno = op_errno;
-        if ((op_ret == -1) && (op_errno != ENOENT)) {
+        if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
                 gf_log (this->name, GF_LOG_DEBUG,
                         "subvolume %s returned -1 (%s)",
                         prev->this->name, strerror (op_errno));
@@ -130,10 +130,11 @@ int
 dht_file_attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int op_ret, int op_errno, struct iatt *stbuf, dict_t *xdata)
 {
-        uint64_t      tmp_subvol = 0;
+        xlator_t     *subvol = 0;
         dht_local_t  *local = NULL;
         call_frame_t *prev = NULL;
         int           ret = -1;
+        inode_t      *inode = NULL;
 
         GF_VALIDATE_OR_GOTO ("dht", frame, err);
         GF_VALIDATE_OR_GOTO ("dht", this, out);
@@ -143,7 +144,7 @@ dht_file_attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         local = frame->local;
         prev = cookie;
 
-        if ((op_ret == -1) && (op_errno != ENOENT)) {
+        if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
                 local->op_errno = op_errno;
                 gf_log (this->name, GF_LOG_DEBUG,
                         "subvolume %s returned -1 (%s)",
@@ -154,21 +155,23 @@ dht_file_attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if (local->call_cnt != 1)
                 goto out;
 
+        local->op_errno = op_errno;
         /* Check if the rebalance phase2 is true */
         if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (stbuf)) {
-                if (local->fd)
-                        ret = fd_ctx_get (local->fd, this, &tmp_subvol);
-                if (ret) {
+                inode = (local->fd) ? local->fd->inode : local->loc.inode;
+                ret = dht_inode_ctx_get1 (this, inode, &subvol);
+                if (!subvol) {
                         /* Phase 2 of migration */
                         local->rebalance.target_op_fn = dht_attr2;
                         ret = dht_rebalance_complete_check (this, frame);
+                        if (!ret)
+                                return 0;
                 } else {
                         /* value is already set in fd_ctx, that means no need
                            to check for whether its complete or not. */
                         dht_attr2 (this, frame, 0);
-                }
-                if (!ret)
                         return 0;
+                }
         }
 
 out:
@@ -381,6 +384,8 @@ dht_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 {
         dht_local_t *local      = NULL;
         int          ret        = 0;
+        inode_t     *inode      = NULL;
+        xlator_t    *subvol = 0;
 
         local = frame->local;
         if (!local) {
@@ -393,22 +398,24 @@ dht_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if (local->call_cnt != 1)
                 goto out;
 
-        if ((op_ret == -1) && (op_errno != ENOENT))
+        if ((op_ret == -1) && !dht_inode_missing(op_errno))
                 goto out;
 
+        local->op_errno = op_errno;
         if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (stbuf)) {
                 /* File would be migrated to other node */
-                ret = fd_ctx_get (local->fd, this, NULL);
-                if (ret) {
+                ret = dht_inode_ctx_get1 (this, inode, &subvol);
+                if (!subvol) {
                         local->rebalance.target_op_fn = dht_readv2;
                         ret = dht_rebalance_complete_check (this, frame);
+                        if (!ret)
+                                return 0;
                 } else {
                         /* value is already set in fd_ctx, that means no need
                            to check for whether its complete or not. */
                         dht_readv2 (this, frame, 0);
-                }
-                if (!ret)
                         return 0;
+                }
         }
 
 out:
@@ -498,14 +505,35 @@ dht_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 {
         int          ret = -1;
         dht_local_t *local = NULL;
+        xlator_t    *subvol = NULL;
+        call_frame_t *prev = NULL;
 
         local = frame->local;
+        prev = cookie;
 
+        if (!prev || !prev->this)
+                goto out;
         if (local->call_cnt != 1)
                 goto out;
+        if ((op_ret == -1) && (op_errno == ENOTCONN) &&
+            IA_ISDIR(local->loc.inode->ia_type)) {
+
+                subvol = dht_subvol_next_available (this, prev->this);
+                if (!subvol)
+                        goto out;
 
-        if ((op_ret == -1) && (op_errno == ENOENT)) {
+                /* check if we are done with visiting every node */
+                if (subvol == local->cached_subvol) {
+                        goto out;
+                }
+
+                STACK_WIND (frame, dht_access_cbk, subvol, subvol->fops->access,
+                            &local->loc, local->rebalance.flags, NULL);
+                return 0;
+        }
+        if ((op_ret == -1) && dht_inode_missing(op_errno)) {
                 /* File would be migrated to other node */
+                local->op_errno = op_errno;
                 local->rebalance.target_op_fn = dht_access2;
                 ret = dht_rebalance_complete_check (frame->this, frame);
                 if (!ret)
@@ -593,8 +621,9 @@ int
 dht_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                int op_ret, int op_errno, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        int           ret = -1;
+        dht_local_t  *local  = NULL;
+        inode_t      *inode  = NULL;
+        xlator_t     *subvol = 0;
 
         local = frame->local;
 
@@ -604,8 +633,8 @@ dht_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
 
         /* If context is set, then send flush() it to the destination */
-        ret = fd_ctx_get (local->fd, this, NULL);
-        if (!ret) {
+        dht_inode_ctx_get1 (this, inode, &subvol);
+        if (subvol) {
                 dht_flush2 (this, frame, 0);
                 return 0;
         }
@@ -621,14 +650,10 @@ dht_flush2 (xlator_t *this, call_frame_t *frame, int op_ret)
 {
         dht_local_t  *local  = NULL;
         xlator_t     *subvol = NULL;
-        uint64_t      tmp_subvol = 0;
-        int           ret = -1;
 
         local = frame->local;
 
-        ret = fd_ctx_get (local->fd, this, &tmp_subvol);
-        if (!ret)
-                subvol = (xlator_t *)(long)tmp_subvol;
+        dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
 
         if (!subvol)
                 subvol = local->cached_subvol;
@@ -690,12 +715,14 @@ dht_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
         dht_local_t  *local = NULL;
         call_frame_t *prev = NULL;
         int           ret = -1;
+        inode_t      *inode = NULL;
+        xlator_t     *subvol = 0;
 
         local = frame->local;
         prev = cookie;
 
         local->op_errno = op_errno;
-        if (op_ret == -1) {
+        if (op_ret == -1 && !dht_inode_missing(op_errno)) {
                 gf_log (this->name, GF_LOG_DEBUG,
                         "subvolume %s returned -1 (%s)",
                         prev->this->name, strerror (op_errno));
@@ -710,8 +737,9 @@ dht_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
                 goto out;
         }
 
-        ret = fd_ctx_get (local->fd, this, NULL);
-        if (ret) {
+        local->op_errno = op_errno;
+        dht_inode_ctx_get1 (this, inode, &subvol);
+        if (!subvol) {
                 local->rebalance.target_op_fn = dht_fsync2;
 
                 /* Check if the rebalance phase1 is true */
@@ -726,11 +754,12 @@ dht_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
                 if (IS_DHT_MIGRATION_PHASE2 (postbuf)) {
                         ret = dht_rebalance_complete_check (this, frame);
                 }
+                if (!ret)
+                        return 0;
         } else {
                 dht_fsync2 (this, frame, 0);
-        }
-        if (!ret)
                 return 0;
+        }
 
 out:
         DHT_STRIP_PHASE1_FLAGS (postbuf);
@@ -746,15 +775,10 @@ dht_fsync2 (xlator_t *this, call_frame_t *frame, int op_ret)
 {
         dht_local_t  *local  = NULL;
         xlator_t     *subvol = NULL;
-        uint64_t      tmp_subvol = 0;
-        int           ret = -1;
 
         local = frame->local;
 
-        ret = fd_ctx_get (local->fd, this, &tmp_subvol);
-        if (!ret)
-                subvol = (xlator_t *)(long)tmp_subvol;
-
+        dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
         if (!subvol)
                 subvol = local->cached_subvol;
 
diff --git a/xlators/cluster/dht/src/dht-inode-write.c b/xlators/cluster/dht/src/dht-inode-write.c
index d4a3ecc39..363bff3bf 100644
--- a/xlators/cluster/dht/src/dht-inode-write.c
+++ b/xlators/cluster/dht/src/dht-inode-write.c
@@ -19,6 +19,9 @@
 int dht_writev2 (xlator_t *this, call_frame_t *frame, int ret);
 int dht_truncate2 (xlator_t *this, call_frame_t *frame, int ret);
 int dht_setattr2 (xlator_t *this, call_frame_t *frame, int ret);
+int dht_fallocate2(xlator_t *this, call_frame_t *frame, int op_ret);
+int dht_discard2(xlator_t *this, call_frame_t *frame, int op_ret);
+int dht_zerofill2(xlator_t *this, call_frame_t *frame, int op_ret);
 
 int
 dht_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
@@ -27,8 +30,9 @@ dht_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 {
         dht_local_t *local = NULL;
         int          ret   = -1;
+        xlator_t    *subvol = NULL;
 
-        if (op_ret == -1) {
+        if (op_ret == -1 && !dht_inode_missing(op_errno)) {
                 goto out;
         }
 
@@ -50,6 +54,7 @@ dht_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         local->rebalance.target_op_fn = dht_writev2;
 
+        local->op_errno = op_errno;
         /* Phase 2 of migration */
         if (IS_DHT_MIGRATION_PHASE2 (postbuf)) {
                 ret = dht_rebalance_complete_check (this, frame);
@@ -62,8 +67,8 @@ dht_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
                 dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
 
-                ret = fd_ctx_get (local->fd, this, NULL);
-                if (!ret) {
+                ret = dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
+                if (subvol) {
                         dht_writev2 (this, frame, 0);
                         return 0;
                 }
@@ -87,14 +92,10 @@ dht_writev2 (xlator_t *this, call_frame_t *frame, int op_ret)
 {
         dht_local_t  *local  = NULL;
         xlator_t     *subvol = NULL;
-        uint64_t      tmp_subvol = 0;
-        int           ret = -1;
 
         local = frame->local;
 
-        ret = fd_ctx_get (local->fd, this, &tmp_subvol);
-        if (!ret)
-                subvol = (xlator_t *)(long)tmp_subvol;
+        dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
 
         if (!subvol)
                 subvol = local->cached_subvol;
@@ -169,6 +170,8 @@ dht_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         dht_local_t  *local = NULL;
         call_frame_t *prev = NULL;
         int           ret = -1;
+        xlator_t    *subvol = NULL;
+        inode_t      *inode = NULL;
 
         GF_VALIDATE_OR_GOTO ("dht", frame, err);
         GF_VALIDATE_OR_GOTO ("dht", this, out);
@@ -178,7 +181,7 @@ dht_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         local = frame->local;
         prev = cookie;
 
-        if ((op_ret == -1) && (op_errno != ENOENT)) {
+        if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
                 local->op_errno = op_errno;
                 local->op_ret = -1;
                 gf_log (this->name, GF_LOG_DEBUG,
@@ -198,6 +201,7 @@ dht_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         local->rebalance.target_op_fn = dht_truncate2;
 
+        local->op_errno = op_errno;
         /* Phase 2 of migration */
         if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) {
                 ret = dht_rebalance_complete_check (this, frame);
@@ -209,8 +213,9 @@ dht_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if (IS_DHT_MIGRATION_PHASE1 (postbuf)) {
                 dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
                 dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
-                ret = fd_ctx_get (local->fd, this, NULL);
-                if (!ret) {
+                inode = (local->fd) ? local->fd->inode : local->loc.inode;
+                dht_inode_ctx_get1 (this, inode, &subvol);
+                if (subvol) {
                         dht_truncate2 (this, frame, 0);
                         return 0;
                 }
@@ -234,16 +239,13 @@ dht_truncate2 (xlator_t *this, call_frame_t *frame, int op_ret)
 {
         dht_local_t  *local  = NULL;
         xlator_t     *subvol = NULL;
-        uint64_t      tmp_subvol = 0;
-        int           ret = -1;
+        inode_t      *inode = NULL;
 
         local = frame->local;
 
-        if (local->fd)
-                ret = fd_ctx_get (local->fd, this, &tmp_subvol);
-        if (!ret)
-                subvol = (xlator_t *)(long)tmp_subvol;
+        inode = local->fd ? local->fd->inode : local->loc.inode;
 
+        dht_inode_ctx_get1 (this, inode, &subvol);
         if (!subvol)
                 subvol = local->cached_subvol;
 
@@ -346,6 +348,407 @@ err:
         return 0;
 }
 
+
+int
+dht_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int op_ret, int op_errno, struct iatt *prebuf,
+                  struct iatt *postbuf, dict_t *xdata)
+{
+        dht_local_t  *local = NULL;
+        call_frame_t *prev = NULL;
+        int           ret = -1;
+        xlator_t    *subvol = NULL;
+
+        GF_VALIDATE_OR_GOTO ("dht", frame, err);
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
+        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
+
+        local = frame->local;
+        prev = cookie;
+
+        if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
+                local->op_errno = op_errno;
+                local->op_ret = -1;
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "subvolume %s returned -1 (%s)",
+                        prev->this->name, strerror (op_errno));
+
+                goto out;
+        }
+
+        if (local->call_cnt != 1) {
+                if (local->stbuf.ia_blocks) {
+                        dht_iatt_merge (this, postbuf, &local->stbuf, NULL);
+                        dht_iatt_merge (this, prebuf, &local->prebuf, NULL);
+                }
+                goto out;
+        }
+        local->rebalance.target_op_fn = dht_fallocate2;
+
+        /* Phase 2 of migration */
+        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) {
+                ret = dht_rebalance_complete_check (this, frame);
+                if (!ret)
+                        return 0;
+        }
+
+        /* Check if the rebalance phase1 is true */
+        if (IS_DHT_MIGRATION_PHASE1 (postbuf)) {
+                dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
+                dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
+                dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
+                if (subvol) {
+                        dht_fallocate2 (this, frame, 0);
+                        return 0;
+                }
+                ret = dht_rebalance_in_progress_check (this, frame);
+                if (!ret)
+                        return 0;
+        }
+
+out:
+        DHT_STRIP_PHASE1_FLAGS (postbuf);
+        DHT_STRIP_PHASE1_FLAGS (prebuf);
+        DHT_STACK_UNWIND (fallocate, frame, op_ret, op_errno,
+                          prebuf, postbuf, xdata);
+err:
+        return 0;
+}
+
+int
+dht_fallocate2(xlator_t *this, call_frame_t *frame, int op_ret)
+{
+        dht_local_t  *local  = NULL;
+        xlator_t     *subvol = NULL;
+
+        local = frame->local;
+
+        dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
+
+        if (!subvol)
+                subvol = local->cached_subvol;
+
+        local->call_cnt = 2; /* This is the second attempt */
+
+	STACK_WIND(frame, dht_fallocate_cbk, subvol, subvol->fops->fallocate,
+		   local->fd, local->rebalance.flags, local->rebalance.offset,
+		   local->rebalance.size, NULL);
+
+        return 0;
+}
+
+int
+dht_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+	      off_t offset, size_t len, dict_t *xdata)
+{
+        xlator_t     *subvol = NULL;
+        int           op_errno = -1;
+        dht_local_t  *local = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+
+        local = dht_local_init (frame, NULL, fd, GF_FOP_FALLOCATE);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+	local->rebalance.flags = mode;
+        local->rebalance.offset = offset;
+	local->rebalance.size = len;
+
+        local->call_cnt = 1;
+        subvol = local->cached_subvol;
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for fd=%p", fd);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        STACK_WIND (frame, dht_fallocate_cbk,
+                    subvol, subvol->fops->fallocate,
+                    fd, mode, offset, len, xdata);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (fallocate, frame, -1, op_errno, NULL, NULL, NULL);
+
+        return 0;
+}
+
+
+int
+dht_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int op_ret, int op_errno, struct iatt *prebuf,
+                struct iatt *postbuf, dict_t *xdata)
+{
+        dht_local_t  *local = NULL;
+        call_frame_t *prev = NULL;
+        int           ret = -1;
+        xlator_t    *subvol = NULL;
+
+        GF_VALIDATE_OR_GOTO ("dht", frame, err);
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
+        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
+
+        local = frame->local;
+        prev = cookie;
+
+        if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
+                local->op_errno = op_errno;
+                local->op_ret = -1;
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "subvolume %s returned -1 (%s)",
+                        prev->this->name, strerror (op_errno));
+
+                goto out;
+        }
+
+        if (local->call_cnt != 1) {
+                if (local->stbuf.ia_blocks) {
+                        dht_iatt_merge (this, postbuf, &local->stbuf, NULL);
+                        dht_iatt_merge (this, prebuf, &local->prebuf, NULL);
+                }
+                goto out;
+        }
+        local->rebalance.target_op_fn = dht_discard2;
+
+        /* Phase 2 of migration */
+        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) {
+                ret = dht_rebalance_complete_check (this, frame);
+                if (!ret)
+                        return 0;
+        }
+
+        /* Check if the rebalance phase1 is true */
+        if (IS_DHT_MIGRATION_PHASE1 (postbuf)) {
+                dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
+                dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
+                dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
+                if (subvol) {
+                        dht_discard2 (this, frame, 0);
+                        return 0;
+                }
+                ret = dht_rebalance_in_progress_check (this, frame);
+                if (!ret)
+                        return 0;
+        }
+
+out:
+        DHT_STRIP_PHASE1_FLAGS (postbuf);
+        DHT_STRIP_PHASE1_FLAGS (prebuf);
+        DHT_STACK_UNWIND (discard, frame, op_ret, op_errno,
+                          prebuf, postbuf, xdata);
+err:
+        return 0;
+}
+
+int
+dht_discard2(xlator_t *this, call_frame_t *frame, int op_ret)
+{
+        dht_local_t  *local  = NULL;
+        xlator_t     *subvol = NULL;
+
+        local = frame->local;
+
+        dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
+
+        if (!subvol)
+                subvol = local->cached_subvol;
+
+        local->call_cnt = 2; /* This is the second attempt */
+
+	STACK_WIND(frame, dht_discard_cbk, subvol, subvol->fops->discard,
+		   local->fd, local->rebalance.offset, local->rebalance.size,
+		   NULL);
+
+        return 0;
+}
+
+int
+dht_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+	    size_t len, dict_t *xdata)
+{
+        xlator_t     *subvol = NULL;
+        int           op_errno = -1;
+        dht_local_t  *local = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+
+        local = dht_local_init (frame, NULL, fd, GF_FOP_DISCARD);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        local->rebalance.offset = offset;
+	local->rebalance.size = len;
+
+        local->call_cnt = 1;
+        subvol = local->cached_subvol;
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for fd=%p", fd);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        STACK_WIND (frame, dht_discard_cbk, subvol, subvol->fops->discard,
+                    fd, offset, len, xdata);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (discard, frame, -1, op_errno, NULL, NULL, NULL);
+
+        return 0;
+}
+
+int
+dht_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int op_ret, int op_errno, struct iatt *prebuf,
+                struct iatt *postbuf, dict_t *xdata)
+{
+        dht_local_t  *local = NULL;
+        call_frame_t *prev  = NULL;
+        int           ret   = -1;
+
+        GF_VALIDATE_OR_GOTO ("dht", frame, err);
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
+        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
+
+        local = frame->local;
+        prev = cookie;
+
+        if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
+                local->op_errno = op_errno;
+                local->op_ret = -1;
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "subvolume %s returned -1 (%s)",
+                        prev->this->name, strerror (op_errno));
+                goto out;
+        }
+
+        if (local->call_cnt != 1) {
+                if (local->stbuf.ia_blocks) {
+                        dht_iatt_merge (this, postbuf, &local->stbuf, NULL);
+                        dht_iatt_merge (this, prebuf, &local->prebuf, NULL);
+                }
+                goto out;
+        }
+        local->rebalance.target_op_fn = dht_zerofill2;
+        /* Phase 2 of migration */
+        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) {
+                ret = dht_rebalance_complete_check (this, frame);
+                if (!ret)
+                        return 0;
+        }
+
+        /* Check if the rebalance phase1 is true */
+        if (IS_DHT_MIGRATION_PHASE1 (postbuf)) {
+                dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
+                dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
+                ret = fd_ctx_get (local->fd, this, NULL);
+                if (!ret) {
+                        dht_zerofill2 (this, frame, 0);
+                        return 0;
+                }
+                ret = dht_rebalance_in_progress_check (this, frame);
+                if (!ret)
+                        return 0;
+        }
+
+out:
+        DHT_STRIP_PHASE1_FLAGS (postbuf);
+        DHT_STRIP_PHASE1_FLAGS (prebuf);
+        DHT_STACK_UNWIND (zerofill, frame, op_ret, op_errno,
+                          prebuf, postbuf, xdata);
+err:
+        return 0;
+}
+
+int
+dht_zerofill2(xlator_t *this, call_frame_t *frame, int op_ret)
+{
+        dht_local_t  *local          = NULL;
+        xlator_t     *subvol         = NULL;
+        uint64_t      tmp_subvol     = 0;
+        int           ret            = -1;
+
+        local = frame->local;
+
+        if (local->fd)
+                ret = fd_ctx_get (local->fd, this, &tmp_subvol);
+        if (!ret)
+                subvol = (xlator_t *)(long)tmp_subvol;
+
+        if (!subvol)
+                subvol = local->cached_subvol;
+
+        local->call_cnt = 2; /* This is the second attempt */
+
+        STACK_WIND(frame, dht_zerofill_cbk, subvol, subvol->fops->zerofill,
+                   local->fd, local->rebalance.offset, local->rebalance.size,
+                   NULL);
+
+        return 0;
+}
+
+int
+dht_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+            off_t len, dict_t *xdata)
+{
+        xlator_t     *subvol       = NULL;
+        int           op_errno     = -1;
+        dht_local_t  *local        = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+
+        local = dht_local_init (frame, NULL, fd, GF_FOP_ZEROFILL);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        local->rebalance.offset = offset;
+        local->rebalance.size = len;
+
+        local->call_cnt = 1;
+        subvol = local->cached_subvol;
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for fd=%p", fd);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        STACK_WIND (frame, dht_zerofill_cbk, subvol, subvol->fops->zerofill,
+                    fd, offset, len, xdata);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (zerofill, frame, -1, op_errno, NULL, NULL, NULL);
+
+        return 0;
+}
+
+
+
 /* handle cases of migration here for 'setattr()' calls */
 int
 dht_file_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
@@ -360,7 +763,7 @@ dht_file_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         prev = cookie;
 
         local->op_errno = op_errno;
-        if ((op_ret == -1) && (op_errno != ENOENT)) {
+        if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
                 gf_log (this->name, GF_LOG_DEBUG,
                         "subvolume %s returned -1 (%s)",
                         prev->this->name, strerror (op_errno));
@@ -397,15 +800,13 @@ dht_setattr2 (xlator_t *this, call_frame_t *frame, int op_ret)
 {
         dht_local_t  *local  = NULL;
         xlator_t     *subvol = NULL;
-        uint64_t      tmp_subvol = 0;
-        int           ret = -1;
+        inode_t      *inode = NULL;
 
         local = frame->local;
 
-        if (local->fd)
-                ret = fd_ctx_get (local->fd, this, &tmp_subvol);
-        if (!ret)
-                subvol = (xlator_t *)(long)tmp_subvol;
+        inode = (local->fd) ? local->fd->inode : local->loc.inode;
+
+        dht_inode_ctx_get1 (this, inode, &subvol);
 
         if (!subvol)
                 subvol = local->cached_subvol;
diff --git a/xlators/cluster/dht/src/dht-layout.c b/xlators/cluster/dht/src/dht-layout.c
index 08697d042..e1a37b77c 100644
--- a/xlators/cluster/dht/src/dht-layout.c
+++ b/xlators/cluster/dht/src/dht-layout.c
@@ -25,14 +25,12 @@
 
 #define layout_size(cnt) (layout_base_size + (cnt * layout_entry_size))
 
-
 dht_layout_t *
 dht_layout_new (xlator_t *this, int cnt)
 {
         dht_layout_t *layout = NULL;
         dht_conf_t   *conf = NULL;
 
-
         conf = this->private;
 
         layout = GF_CALLOC (1, layout_size (cnt),
@@ -50,6 +48,7 @@ dht_layout_new (xlator_t *this, int cnt)
         }
 
         layout->ref = 1;
+
 out:
         return layout;
 }
@@ -59,9 +58,7 @@ dht_layout_t *
 dht_layout_get (xlator_t *this, inode_t *inode)
 {
         dht_conf_t   *conf = NULL;
-        uint64_t      layout_int = 0;
         dht_layout_t *layout = NULL;
-        int           ret    = -1;
 
         conf = this->private;
         if (!conf)
@@ -69,9 +66,8 @@ dht_layout_get (xlator_t *this, inode_t *inode)
 
         LOCK (&conf->layout_lock);
         {
-                ret = inode_ctx_get (inode, this, &layout_int);
-                if (ret == 0) {
-                        layout = (dht_layout_t *) (unsigned long) layout_int;
+                dht_inode_ctx_layout_get (inode, this, &layout);
+                if (layout) {
                         layout->ref++;
                 }
         }
@@ -89,7 +85,6 @@ dht_layout_set (xlator_t *this, inode_t *inode, dht_layout_t *layout)
         int           oldret = -1;
         int           ret = 0;
         dht_layout_t *old_layout;
-        uint64_t      old_layout_int;
 
         conf = this->private;
         if (!conf)
@@ -97,16 +92,13 @@ dht_layout_set (xlator_t *this, inode_t *inode, dht_layout_t *layout)
 
         LOCK (&conf->layout_lock);
         {
-                oldret = inode_ctx_get (inode, this, &old_layout_int);
-
+                oldret = dht_inode_ctx_layout_get (inode, this, &old_layout);
                 layout->ref++;
-                ret = inode_ctx_put (inode, this, (uint64_t) (unsigned long)
-                                     layout);
+                dht_inode_ctx_layout_set (inode, this, layout);
         }
         UNLOCK (&conf->layout_lock);
 
-        if (oldret == 0) {
-                old_layout = (dht_layout_t *) (unsigned long) old_layout_int;
+        if (!oldret) {
                 dht_layout_unref (this, old_layout);
         }
 
@@ -121,7 +113,7 @@ dht_layout_unref (xlator_t *this, dht_layout_t *layout)
         dht_conf_t  *conf = NULL;
         int          ref = 0;
 
-        if (layout->preset || !this->private)
+        if (!layout || layout->preset || !this->private)
                 return;
 
         conf = this->private;
@@ -165,7 +157,7 @@ dht_layout_search (xlator_t *this, dht_layout_t *layout, const char *name)
         int        ret = 0;
 
 
-        ret = dht_hash_compute (layout->type, name, &hash);
+        ret = dht_hash_compute (this, layout->type, name, &hash);
         if (ret != 0) {
                 gf_log (this->name, GF_LOG_WARNING,
                         "hash computation failed for type=%d name=%s",
@@ -308,9 +300,13 @@ dht_disk_layout_merge (xlator_t *this, dht_layout_t *layout,
                 return -1;
         }
 
-	switch (disk_layout[1]) {
+        type = ntoh32 (disk_layout[1]);
+	switch (type) {
+        case DHT_HASH_TYPE_DM_USER:
+                gf_log (this->name, GF_LOG_DEBUG, "found user-set layout");
+                layout->type = type;
+                /* Fall through. */
 	case DHT_HASH_TYPE_DM:
-		type = ntoh32 (disk_layout[1]);
 		break;
         default:
 		gf_log (this->name, GF_LOG_CRITICAL,
@@ -338,11 +334,12 @@ int
 dht_layout_merge (xlator_t *this, dht_layout_t *layout, xlator_t *subvol,
                   int op_ret, int op_errno, dict_t *xattr)
 {
-        int      i     = 0;
-        int      ret   = -1;
-        int      err   = -1;
-        void    *disk_layout_raw = NULL;
-        int      disk_layout_len = 0;
+        int         i     = 0;
+        int         ret   = -1;
+        int         err   = -1;
+        void       *disk_layout_raw = NULL;
+        int         disk_layout_len = 0;
+        dht_conf_t *conf = this->private;
 
         if (op_ret != 0) {
                 err = op_errno;
@@ -363,12 +360,12 @@ dht_layout_merge (xlator_t *this, dht_layout_t *layout, xlator_t *subvol,
 
         if (xattr) {
                 /* during lookup and not mkdir */
-                ret = dict_get_ptr_and_len (xattr, "trusted.glusterfs.dht",
+                ret = dict_get_ptr_and_len (xattr, conf->xattr_name,
 					    &disk_layout_raw, &disk_layout_len);
         }
 
         if (ret != 0) {
-                layout->list[i].err = -1;
+                layout->list[i].err = 0;
                 gf_log (this->name, GF_LOG_TRACE,
                         "missing disk layout on %s. err = %d",
                         subvol->name, err);
@@ -415,6 +412,22 @@ dht_layout_entry_swap (dht_layout_t *layout, int i, int j)
         layout->list[j].err    = err_swap;
 }
 
+void
+dht_layout_range_swap (dht_layout_t *layout, int i, int j)
+{
+        uint32_t  start_swap = 0;
+        uint32_t  stop_swap = 0;
+
+        start_swap  = layout->list[i].start;
+        stop_swap   = layout->list[i].stop;
+
+        layout->list[i].start  = layout->list[j].start;
+        layout->list[i].stop   = layout->list[j].stop;
+
+        layout->list[j].start  = start_swap;
+        layout->list[j].stop   = stop_swap;
+}
+
 int64_t
 dht_layout_entry_cmp_volname (dht_layout_t *layout, int i, int j)
 {
@@ -422,17 +435,39 @@ dht_layout_entry_cmp_volname (dht_layout_t *layout, int i, int j)
                         layout->list[j].xlator->name));
 }
 
+
+gf_boolean_t
+dht_is_subvol_in_layout (dht_layout_t *layout, xlator_t *xlator)
+{
+        int i = 0;
+
+        for (i = 0; i < layout->cnt; i++) {
+                /* Check if xlator is already part of layout, and layout is
+                 * non-zero. */
+                if (!strcmp (layout->list[i].xlator->name, xlator->name)) {
+                        if (layout->list[i].start != layout->list[i].stop)
+                                return _gf_true;
+                        break;
+                }
+        }
+        return _gf_false;
+}
+
 int64_t
 dht_layout_entry_cmp (dht_layout_t *layout, int i, int j)
 {
         int64_t diff = 0;
 
-        if (layout->list[i].err || layout->list[j].err)
-                diff = layout->list[i].err - layout->list[j].err;
-        else
-                diff = (int64_t) layout->list[i].start
-                        - (int64_t) layout->list[j].start;
+        /* swap zero'ed out layouts to front, if needed */
+        if (!layout->list[j].start && !layout->list[j].stop) {
+                diff = (int64_t) layout->list[i].stop
+                       - (int64_t) layout->list[j].stop;
+                       goto out;
+        }
+        diff = (int64_t) layout->list[i].start
+                - (int64_t) layout->list[j].start;
 
+out:
         return diff;
 }
 
@@ -481,7 +516,8 @@ dht_layout_sort_volname (dht_layout_t *layout)
 int
 dht_layout_anomalies (xlator_t *this, loc_t *loc, dht_layout_t *layout,
                       uint32_t *holes_p, uint32_t *overlaps_p,
-                      uint32_t *missing_p, uint32_t *down_p, uint32_t *misc_p)
+                      uint32_t *missing_p, uint32_t *down_p, uint32_t *misc_p,
+                      uint32_t *no_space_p)
 {
         uint32_t    overlaps = 0;
         uint32_t    missing  = 0;
@@ -494,30 +530,51 @@ dht_layout_anomalies (xlator_t *this, loc_t *loc, dht_layout_t *layout,
         uint32_t    prev_stop = 0;
         uint32_t    last_stop = 0;
         char        is_virgin = 1;
-
-        /* TODO: explain what is happening */
-
+        uint32_t    no_space  = 0;
+
+        /* This funtion scans through the layout spread of a directory to
+           check if there are any anomalies. Prior to calling this function
+           the layout entries should be sorted in the ascending order.
+
+           If the layout entry has err != 0
+                then increment the corresponding anomaly.
+           else
+                if (start of the current layout entry > stop + 1 of previous
+                   non erroneous layout entry)
+                        then it indicates a hole in the layout
+                if (start of the current layout entry < stop + 1 of previous
+                    non erroneous layout entry)
+                         then it indicates an overlap in the layout
+        */
         last_stop = layout->list[0].start - 1;
         prev_stop = last_stop;
 
         for (i = 0; i < layout->cnt; i++) {
-                if (layout->list[i].err) {
-                        switch (layout->list[i].err) {
-                        case -1:
-                        case ENOENT:
-                                missing++;
-                                break;
-                        case ENOTCONN:
-                                down++;
-                                break;
-                        case ENOSPC:
-                                down++;
-                                break;
-                        default:
-                                misc++;
+                switch (layout->list[i].err) {
+                case -1:
+                case ENOENT:
+                case ESTALE:
+                        missing++;
+                        continue;
+                case ENOTCONN:
+                        down++;
+                        continue;
+                case ENOSPC:
+                        no_space++;
+                        continue;
+                case 0:
+                        /* if err == 0 and start == stop, then it is a non misc++;
+                         * participating subvolume(spread-cnt). Then, do not
+                         * check for anomalies. If start != stop, then treat it
+                         * as misc err */
+                        if (layout->list[i].start == layout->list[i].stop) {
+                                continue;
                         }
+                        break;
+                default:
+                        misc++;
                         continue;
-                }
+                 }
 
                 is_virgin = 0;
 
@@ -550,6 +607,9 @@ dht_layout_anomalies (xlator_t *this, loc_t *loc, dht_layout_t *layout,
         if (misc_p)
                 *misc_p = misc;
 
+        if (no_space_p)
+                *no_space_p = no_space;
+
         return ret;
 }
 
@@ -574,7 +634,7 @@ dht_layout_normalize (xlator_t *this, loc_t *loc, dht_layout_t *layout)
 
         ret = dht_layout_anomalies (this, loc, layout,
                                     &holes, &overlaps,
-                                    &missing, &down, &misc);
+                                    &missing, &down, &misc, NULL);
         if (ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING,
                         "error while finding anomalies in %s -- not good news",
@@ -618,21 +678,30 @@ out:
         return ret;
 }
 
+int
+dht_dir_has_layout (dict_t *xattr, char *name)
+{
+
+        void     *disk_layout_raw = NULL;
+
+        return dict_get_ptr (xattr, name, &disk_layout_raw);
+}
 
 int
 dht_layout_dir_mismatch (xlator_t *this, dht_layout_t *layout, xlator_t *subvol,
                          loc_t *loc, dict_t *xattr)
 {
-        int       idx = 0;
-        int       pos = -1;
-        int       ret = 0;
-        int       err = 0;
-        int       dict_ret = 0;
-        int32_t   disk_layout[4];
-        void     *disk_layout_raw = NULL;
-        int32_t   count = -1;
-        uint32_t  start_off = -1;
-        uint32_t  stop_off = -1;
+        int         idx = 0;
+        int         pos = -1;
+        int         ret = 0;
+        int         err = 0;
+        int         dict_ret = 0;
+        int32_t     disk_layout[4];
+        void       *disk_layout_raw = NULL;
+        int32_t     count = -1;
+        uint32_t    start_off = -1;
+        uint32_t    stop_off = -1;
+        dht_conf_t *conf = this->private;
 
 
         for (idx = 0; idx < layout->cnt; idx++) {
@@ -662,11 +731,11 @@ dht_layout_dir_mismatch (xlator_t *this, dht_layout_t *layout, xlator_t *subvol,
                 goto out;
         }
 
-        dict_ret = dict_get_ptr (xattr, "trusted.glusterfs.dht",
+        dict_ret = dict_get_ptr (xattr, conf->xattr_name,
                                  &disk_layout_raw);
 
         if (dict_ret < 0) {
-                if (err == 0) {
+                if (err == 0 && layout->list[pos].stop) {
                         gf_log (this->name, GF_LOG_INFO,
                                 "%s - disk layout missing", loc->path);
                         ret = -1;
@@ -727,7 +796,7 @@ dht_layout_preset (xlator_t *this, xlator_t *subvol, inode_t *inode)
 
         LOCK (&conf->layout_lock);
         {
-                inode_ctx_put (inode, this, (uint64_t)(long)layout);
+                dht_inode_ctx_layout_set (inode, this, layout);
         }
         UNLOCK (&conf->layout_lock);
 
diff --git a/xlators/cluster/dht/src/dht-linkfile.c b/xlators/cluster/dht/src/dht-linkfile.c
index 9de370341..dbc9d0b3c 100644
--- a/xlators/cluster/dht/src/dht-linkfile.c
+++ b/xlators/cluster/dht/src/dht-linkfile.c
@@ -19,6 +19,35 @@
 #include "compat.h"
 #include "dht-common.h"
 
+int
+dht_linkfile_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                         int op_ret, int op_errno,
+                         inode_t *inode, struct iatt *stbuf, dict_t *xattr,
+                         struct iatt *postparent)
+{
+        char          is_linkfile   = 0;
+        dht_conf_t   *conf          = NULL;
+        dht_local_t  *local         = NULL;
+        call_frame_t *prev          = NULL;
+
+        local = frame->local;
+        prev = cookie;
+        conf = this->private;
+
+        if (op_ret)
+                goto out;
+
+        is_linkfile = check_is_linkfile (inode, stbuf, xattr,
+                                         conf->link_xattr_name);
+        if (!is_linkfile)
+                gf_log (this->name, GF_LOG_WARNING, "got non-linkfile %s:%s",
+                        prev->this->name, local->loc.path);
+out:
+        local->linkfile.linkfile_cbk (frame, cookie, this, op_ret, op_errno,
+                                      inode, stbuf, postparent, postparent,
+                                      xattr);
+        return 0;
+}
 
 #define is_equal(a, b) (a == b)
 int
@@ -28,6 +57,11 @@ dht_linkfile_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                          struct iatt *postparent, dict_t *xdata)
 {
         dht_local_t  *local = NULL;
+        xlator_t     *subvol = NULL;
+        call_frame_t *prev = NULL;
+        dict_t       *xattrs = NULL;
+        dht_conf_t   *conf = NULL;
+        int           ret = -1;
 
         local = frame->local;
 
@@ -36,21 +70,48 @@ dht_linkfile_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         FRAME_SU_UNDO (frame, dht_local_t);
 
+        if (op_ret && (op_errno == EEXIST)) {
+                conf = this->private;
+                prev = cookie;
+                subvol = prev->this;
+                if (!subvol)
+                        goto out;
+                xattrs = dict_new ();
+                if (!xattrs)
+                        goto out;
+                ret = dict_set_uint32 (xattrs, conf->link_xattr_name, 256);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to set linkto key");
+                        goto out;
+                }
+
+                STACK_WIND (frame, dht_linkfile_lookup_cbk, subvol,
+                            subvol->fops->lookup, &local->loc, xattrs);
+                if (xattrs)
+                        dict_unref (xattrs);
+                return 0;
+        }
+out:
         local->linkfile.linkfile_cbk (frame, cookie, this, op_ret, op_errno,
                                       inode, stbuf, preparent, postparent,
                                       xdata);
+        if (xattrs)
+                dict_unref (xattrs);
         return 0;
 }
 
 
 int
 dht_linkfile_create (call_frame_t *frame, fop_mknod_cbk_t linkfile_cbk,
+                     xlator_t *this,
                      xlator_t *tovol, xlator_t *fromvol, loc_t *loc)
 {
         dht_local_t *local = NULL;
         dict_t      *dict = NULL;
         int          need_unref = 0;
         int          ret = 0;
+        dht_conf_t  *conf = this->private;
 
         local = frame->local;
         local->linkfile.linkfile_cbk = linkfile_cbk;
@@ -78,8 +139,7 @@ dht_linkfile_create (call_frame_t *frame, fop_mknod_cbk_t linkfile_cbk,
                 gf_log ("dht-linkfile", GF_LOG_INFO,
                         "%s: internal-fop set failed", loc->path);
 
-        ret = dict_set_str (dict, "trusted.glusterfs.dht.linkto",
-                                   tovol->name);
+        ret = dict_set_str (dict, conf->link_xattr_name, tovol->name);
 
         if (ret < 0) {
                 gf_log (frame->this->name, GF_LOG_INFO,
@@ -184,7 +244,7 @@ dht_linkfile_subvol (xlator_t *this, inode_t *inode, struct iatt *stbuf,
         if (!xattr)
                 goto out;
 
-        ret = dict_get_ptr (xattr, "trusted.glusterfs.dht.linkto", &volname);
+        ret = dict_get_ptr (xattr, conf->link_xattr_name, &volname);
 
         if ((-1 == ret) || !volname)
                 goto out;
@@ -237,11 +297,11 @@ dht_linkfile_attr_heal (call_frame_t *frame, xlator_t *this)
         GF_VALIDATE_OR_GOTO ("dht", local, out);
         GF_VALIDATE_OR_GOTO ("dht", local->link_subvol, out);
 
-        if ((local->stbuf.ia_type == IA_INVAL) ||
-            (is_equal (frame->root->uid, local->stbuf.ia_uid) &&
-             is_equal (frame->root->gid, local->stbuf.ia_gid)))
+        if (local->stbuf.ia_type == IA_INVAL)
                 return 0;
 
+        uuid_copy (local->loc.gfid, local->stbuf.ia_gfid);
+
         copy = copy_frame (frame);
 
         if (!copy)
diff --git a/xlators/cluster/dht/src/dht-mem-types.h b/xlators/cluster/dht/src/dht-mem-types.h
index 4f3a90e91..e893eb48f 100644
--- a/xlators/cluster/dht/src/dht-mem-types.h
+++ b/xlators/cluster/dht/src/dht-mem-types.h
@@ -28,6 +28,8 @@ enum gf_dht_mem_types_ {
         gf_dht_mt_subvol_time,
         gf_dht_mt_loc_t,
         gf_defrag_info_mt,
+        gf_dht_mt_inode_ctx_t,
+        gf_dht_mt_ctx_stat_time_t,
         gf_dht_mt_end
 };
 #endif
diff --git a/xlators/cluster/dht/src/dht-rebalance.c b/xlators/cluster/dht/src/dht-rebalance.c
index 6ea2ffbde..4f78f5203 100644
--- a/xlators/cluster/dht/src/dht-rebalance.c
+++ b/xlators/cluster/dht/src/dht-rebalance.c
@@ -16,6 +16,8 @@
 
 #include "dht-common.h"
 #include "xlator.h"
+#include <signal.h>
+#include <fnmatch.h>
 
 #define GF_DISK_SECTOR_SIZE             512
 #define DHT_REBALANCE_PID               4242 /* Change it if required */
@@ -56,7 +58,8 @@ dht_write_with_holes (xlator_t *to, fd_t *fd, struct iovec *vec, int count,
                                 if (ret < 0) {
                                         gf_log (THIS->name, GF_LOG_WARNING,
                                                 "failed to write (%s)",
-                                                strerror (errno));
+                                                strerror (-ret));
+                                        ret = -1;
                                         goto out;
                                 }
 
@@ -74,7 +77,8 @@ dht_write_with_holes (xlator_t *to, fd_t *fd, struct iovec *vec, int count,
                                 /* 'path' will be logged in calling function */
                                 gf_log (THIS->name, GF_LOG_WARNING,
                                         "failed to write (%s)",
-                                        strerror (errno));
+                                        strerror (-ret));
+                                ret = -1;
                                 goto out;
                         }
                 }
@@ -90,6 +94,41 @@ out:
 
 }
 
+/*
+   return values:
+   -1 : failure
+   -2 : success
+
+Hard link migration is carried out in three stages.
+
+(Say there are n hardlinks)
+Stage 1: Setting the new hashed subvol information on the 1st hardlink
+         encountered (linkto setxattr)
+
+Stage 2: Creating hardlinks on new hashed subvol for the 2nd to (n-1)th
+         hardlink
+
+Stage 3: Physical migration of the data file for nth hardlink
+
+Why to deem "-2" as success and not "0":
+
+   dht_migrate_file expects return value "0" from _is_file_migratable if
+the file has to be migrated.
+
+   _is_file_migratable returns zero only when it is called with the
+flag "GF_DHT_MIGRATE_HARDLINK_IN_PROGRESS".
+
+   gf_defrag_handle_hardlink calls dht_migrate_file for physical migration
+of the data file with the flag "GF_DHT_MIGRATE_HARDLINK_IN_PROGRESS"
+
+Hence, gf_defrag_handle_hardlink returning "0" for success will force
+"dht_migrate_file" to migrate each of the hardlink which is not intended.
+
+For each of the three stage mentioned above "-2" will be returned and will
+be converted to "0" in dht_migrate_file.
+
+*/
+
 int32_t
 gf_defrag_handle_hardlink (xlator_t *this, loc_t *loc, dict_t  *xattrs,
                            struct iatt *stbuf)
@@ -101,12 +140,16 @@ gf_defrag_handle_hardlink (xlator_t *this, loc_t *loc, dict_t  *xattrs,
         data_t                 *data            = NULL;
         struct iatt             iatt            = {0,};
         int32_t                 op_errno        = 0;
+        dht_conf_t             *conf            = NULL;
 
         GF_VALIDATE_OR_GOTO ("defrag", loc, out);
         GF_VALIDATE_OR_GOTO ("defrag", loc->name, out);
         GF_VALIDATE_OR_GOTO ("defrag", stbuf, out);
         GF_VALIDATE_OR_GOTO ("defrag", this, out);
         GF_VALIDATE_OR_GOTO ("defrag", xattrs, out);
+        GF_VALIDATE_OR_GOTO ("defrag", this->private, out);
+
+        conf = this->private;
 
         if (uuid_is_null (loc->pargfid)) {
                 gf_log ("", GF_LOG_ERROR, "loc->pargfid is NULL for "
@@ -137,10 +180,10 @@ gf_defrag_handle_hardlink (xlator_t *this, loc_t *loc, dict_t  *xattrs,
         gf_log (this->name, GF_LOG_INFO, "Attempting to migrate hardlink %s "
                 "with gfid %s from %s -> %s", loc->name, uuid_utoa (loc->gfid),
                 cached_subvol->name, hashed_subvol->name);
-        data = dict_get (xattrs, DHT_LINKFILE_KEY);
+        data = dict_get (xattrs, conf->link_xattr_name);
         /* set linkto on cached -> hashed if not present, else link it */
         if (!data) {
-                ret = dict_set_str (xattrs, DHT_LINKFILE_KEY,
+                ret = dict_set_str (xattrs, conf->link_xattr_name,
                                     hashed_subvol->name);
                 if (ret) {
                         gf_log (this->name, GF_LOG_ERROR, "Failed to set "
@@ -152,9 +195,11 @@ gf_defrag_handle_hardlink (xlator_t *this, loc_t *loc, dict_t  *xattrs,
                 if (ret) {
                         gf_log (this->name, GF_LOG_ERROR, "Linkto setxattr "
                                 "failed %s -> %s (%s)", cached_subvol->name,
-                                loc->name, strerror (errno));
+                                loc->name, strerror (-ret));
+                        ret = -1;
                         goto out;
                 }
+                ret = -2;
                 goto out;
         } else {
                 linkto_subvol = dht_linkfile_subvol (this, NULL, NULL, xattrs);
@@ -167,7 +212,8 @@ gf_defrag_handle_hardlink (xlator_t *this, loc_t *loc, dict_t  *xattrs,
 
                 ret = syncop_link (hashed_subvol, loc, loc);
                 if  (ret) {
-                        op_errno = errno;
+                        op_errno = -ret;
+                        ret = -1;
                         gf_log (this->name, GF_LOG_ERROR, "link of %s -> %s"
                                 " failed on  subvol %s (%s)", loc->name,
                                 uuid_utoa(loc->gfid),
@@ -179,7 +225,8 @@ gf_defrag_handle_hardlink (xlator_t *this, loc_t *loc, dict_t  *xattrs,
         ret = syncop_lookup (hashed_subvol, loc, NULL, &iatt, NULL, NULL);
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR, "Failed lookup %s on %s (%s)"
-                        , loc->name, hashed_subvol->name, strerror (errno));
+                        , loc->name, hashed_subvol->name, strerror (-ret));
+                ret = -1;
                 goto out;
         }
 
@@ -189,12 +236,19 @@ gf_defrag_handle_hardlink (xlator_t *this, loc_t *loc, dict_t  *xattrs,
                 if (ret)
                         goto out;
         }
-        ret = 0;
+        ret = -2;
 out:
         return ret;
 }
 
-
+/*
+     return values
+     0 : File will be migrated
+    -2 : File will not be migrated
+         (This is the return value from gf_defrag_handle_hardlink. Checkout
+         gf_defrag_handle_hardlink for description of "returning -2")
+    -1 : failure
+*/
 static inline int
 __is_file_migratable (xlator_t *this, loc_t *loc,
                       struct iatt *stbuf, dict_t *xattrs, int flags)
@@ -217,7 +271,12 @@ __is_file_migratable (xlator_t *this, loc_t *loc,
                 if (flags == GF_DHT_MIGRATE_HARDLINK) {
                         ret = gf_defrag_handle_hardlink (this, loc,
                                                          xattrs, stbuf);
-                        if (ret) {
+
+                   /*
+                   Returning zero will force the file to be remigrated.
+                   Checkout gf_defrag_handle_hardlink for more information.
+                   */
+                        if (ret && ret != -2) {
                                 gf_log (this->name, GF_LOG_WARNING,
                                         "%s: failed to migrate file with link",
                                         loc->path);
@@ -225,8 +284,8 @@ __is_file_migratable (xlator_t *this, loc_t *loc,
                 } else {
                         gf_log (this->name, GF_LOG_WARNING,
                                 "%s: file has hardlinks", loc->path);
+                        ret = -ENOTSUP;
                 }
-                ret = ENOTSUP;
                 goto out;
         }
 
@@ -238,14 +297,16 @@ out:
 
 static inline int
 __dht_rebalance_create_dst_file (xlator_t *to, xlator_t *from, loc_t *loc, struct iatt *stbuf,
-                                 dict_t *dict, fd_t **dst_fd)
+                                 dict_t *dict, fd_t **dst_fd, dict_t *xattr)
 {
-        xlator_t *this = NULL;
-        int       ret  = -1;
-        fd_t     *fd   = NULL;
-        struct iatt new_stbuf = {0,};
+        xlator_t    *this = NULL;
+        int          ret  = -1;
+        fd_t        *fd   = NULL;
+        struct iatt  new_stbuf = {0,};
+        dht_conf_t  *conf = NULL;
 
         this = THIS;
+        conf = this->private;
 
         ret = dict_set_static_bin (dict, "gfid-req", stbuf->ia_gfid, 16);
         if (ret) {
@@ -254,7 +315,7 @@ __dht_rebalance_create_dst_file (xlator_t *to, xlator_t *from, loc_t *loc, struc
                 goto out;
         }
 
-        ret = dict_set_str (dict, DHT_LINKFILE_KEY, from->name);
+        ret = dict_set_str (dict, conf->link_xattr_name, from->name);
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "%s: failed to set gfid in dict for create", loc->path);
@@ -281,32 +342,46 @@ __dht_rebalance_create_dst_file (xlator_t *to, xlator_t *from, loc_t *loc, struc
                         goto out;
                 }
         }
-        if ((ret == -1) && (errno != ENOENT)) {
+        if ((ret < 0) && (-ret != ENOENT)) {
                 /* File exists in destination, but not accessible */
                 gf_log (THIS->name, GF_LOG_WARNING,
                         "%s: failed to lookup file (%s)",
-                        loc->path, strerror (errno));
+                        loc->path, strerror (-ret));
+                ret = -1;
                 goto out;
         }
 
         /* Create the destination with LINKFILE mode, and linkto xattr,
            if the linkfile already exists, it will just open the file */
         ret = syncop_create (to, loc, O_RDWR, DHT_LINKFILE_MODE, fd,
-                             dict);
+                             dict, &new_stbuf);
         if (ret < 0) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "failed to create %s on %s (%s)",
-                        loc->path, to->name, strerror (errno));
+                        loc->path, to->name, strerror (-ret));
+                ret = -1;
                 goto out;
         }
 
+        ret = syncop_fsetxattr (to, fd, xattr, 0);
+        if (ret < 0)
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: failed to set xattr on %s (%s)",
+                        loc->path, to->name, strerror (-ret));
+
+        ret = syncop_ftruncate (to, fd, stbuf->ia_size);
+        if (ret < 0)
+                gf_log (this->name, GF_LOG_ERROR,
+                        "ftruncate failed for %s on %s (%s)",
+                        loc->path, to->name, strerror (-ret));
+
         ret = syncop_fsetattr (to, fd, stbuf,
                                (GF_SET_ATTR_UID | GF_SET_ATTR_GID),
                                 NULL, NULL);
         if (ret < 0)
                 gf_log (this->name, GF_LOG_ERROR,
                         "chown failed for %s on %s (%s)",
-                        loc->path, to->name, strerror (errno));
+                        loc->path, to->name, strerror (-ret));
 
         if (dst_fd)
                 *dst_fd = fd;
@@ -327,13 +402,17 @@ __dht_check_free_space (xlator_t *to, xlator_t *from, loc_t *loc,
         int             ret        = -1;
         xlator_t       *this       = NULL;
 
+        uint64_t        src_statfs_blocks = 1;
+        uint64_t        dst_statfs_blocks = 1;
+
         this = THIS;
 
         ret = syncop_statfs (from, loc, &src_statfs);
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "failed to get statfs of %s on %s (%s)",
-                        loc->path, from->name, strerror (errno));
+                        loc->path, from->name, strerror (-ret));
+                ret = -1;
                 goto out;
         }
 
@@ -341,7 +420,8 @@ __dht_check_free_space (xlator_t *to, xlator_t *from, loc_t *loc,
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "failed to get statfs of %s on %s (%s)",
-                        loc->path, to->name, strerror (errno));
+                        loc->path, to->name, strerror (-ret));
+                ret = -1;
                 goto out;
         }
 
@@ -350,22 +430,34 @@ __dht_check_free_space (xlator_t *to, xlator_t *from, loc_t *loc,
         if (flag != GF_DHT_MIGRATE_DATA)
                 goto check_avail_space;
 
-        if (((dst_statfs.f_bavail *
-              dst_statfs.f_bsize) / GF_DISK_SECTOR_SIZE) <
-            (((src_statfs.f_bavail * src_statfs.f_bsize) /
-              GF_DISK_SECTOR_SIZE) - stbuf->ia_blocks)) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "data movement attempted from node (%s) with"
-                        " higher disk space to a node (%s) with "
-                        "lesser disk space (%s)", from->name,
-                        to->name, loc->path);
-
-                /* this is not a 'failure', but we don't want to
-                   consider this as 'success' too :-/ */
-                ret = 1;
-                goto out;
+        /* Check:
+           During rebalance `migrate-data` - Destination subvol experiences
+           a `reduction` in 'blocks' of free space, at the same time source
+           subvol gains certain 'blocks' of free space. A valid check is
+           necessary here to avoid errorneous move to destination where
+           the space could be scantily available.
+         */
+        if (stbuf) {
+                dst_statfs_blocks = ((dst_statfs.f_bavail *
+                                      dst_statfs.f_bsize) /
+                                     GF_DISK_SECTOR_SIZE);
+                src_statfs_blocks = ((src_statfs.f_bavail *
+                                      src_statfs.f_bsize) /
+                                     GF_DISK_SECTOR_SIZE);
+                if ((dst_statfs_blocks - stbuf->ia_blocks) <
+                    (src_statfs_blocks + stbuf->ia_blocks)) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "data movement attempted from node (%s) with"
+                                " higher disk space to a node (%s) with "
+                                "lesser disk space (%s)", from->name,
+                                to->name, loc->path);
+
+                        /* this is not a 'failure', but we don't want to
+                           consider this as 'success' too :-/ */
+                        ret = 1;
+                        goto out;
+                }
         }
-
 check_avail_space:
         if (((dst_statfs.f_bavail * dst_statfs.f_bsize) /
               GF_DISK_SECTOR_SIZE) < stbuf->ia_blocks) {
@@ -416,8 +508,7 @@ __dht_rebalance_migrate_data (xlator_t *from, xlator_t *to, fd_t *src, fd_t *dst
                 offset += ret;
                 total += ret;
 
-                if (vector)
-                        GF_FREE (vector);
+                GF_FREE (vector);
                 if (iobref)
                         iobref_unref (iobref);
                 iobref = NULL;
@@ -425,11 +516,12 @@ __dht_rebalance_migrate_data (xlator_t *from, xlator_t *to, fd_t *src, fd_t *dst
         }
         if (iobref)
                 iobref_unref (iobref);
-        if (vector)
-                GF_FREE (vector);
+        GF_FREE (vector);
 
         if (ret >= 0)
                 ret = 0;
+        else
+                ret = -1;
 
         return ret;
 }
@@ -444,8 +536,10 @@ __dht_rebalance_open_src_file (xlator_t *from, xlator_t *to, loc_t *loc,
         dict_t      *dict = NULL;
         xlator_t    *this = NULL;
         struct iatt  iatt = {0,};
+        dht_conf_t  *conf = NULL;
 
         this = THIS;
+        conf = this->private;
 
         fd = fd_create (loc->inode, DHT_REBALANCE_PID);
         if (!fd) {
@@ -456,10 +550,11 @@ __dht_rebalance_open_src_file (xlator_t *from, xlator_t *to, loc_t *loc,
         }
 
         ret = syncop_open (from, loc, O_RDWR, fd);
-        if (ret == -1) {
+        if (ret < 0) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "failed to open file %s on %s (%s)",
-                        loc->path, from->name, strerror (errno));
+                        loc->path, from->name, strerror (-ret));
+                ret = -1;
                 goto out;
         }
 
@@ -468,7 +563,7 @@ __dht_rebalance_open_src_file (xlator_t *from, xlator_t *to, loc_t *loc,
         if (!dict)
                 goto out;
 
-        ret = dict_set_str (dict, DHT_LINKFILE_KEY, to->name);
+        ret = dict_set_str (dict, conf->link_xattr_name, to->name);
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "failed to set xattr in dict for %s (linkto:%s)",
@@ -482,7 +577,8 @@ __dht_rebalance_open_src_file (xlator_t *from, xlator_t *to, loc_t *loc,
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "failed to set xattr on %s in %s (%s)",
-                        loc->path, from->name, strerror (errno));
+                        loc->path, from->name, strerror (-ret));
+                ret = -1;
                 goto out;
         }
 
@@ -496,7 +592,8 @@ __dht_rebalance_open_src_file (xlator_t *from, xlator_t *to, loc_t *loc,
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "failed to set mode on %s in %s (%s)",
-                        loc->path, from->name, strerror (errno));
+                        loc->path, from->name, strerror (-ret));
+                ret = -1;
                 goto out;
         }
 
@@ -521,12 +618,13 @@ migrate_special_files (xlator_t *this, xlator_t *from, xlator_t *to, loc_t *loc,
         dict_t      *dict     = NULL;
         char        *link     = NULL;
         struct iatt  stbuf    = {0,};
+        dht_conf_t  *conf     = this->private;
 
         dict = dict_new ();
         if (!dict)
                 goto out;
 
-        ret = dict_set_int32 (dict, DHT_LINKFILE_KEY, 256);
+        ret = dict_set_int32 (dict, conf->link_xattr_name, 256);
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "%s: failed to set 'linkto' key in dict", loc->path);
@@ -535,19 +633,21 @@ migrate_special_files (xlator_t *this, xlator_t *from, xlator_t *to, loc_t *loc,
 
         /* check in the destination if the file is link file */
         ret = syncop_lookup (to, loc, dict, &stbuf, &rsp_dict, NULL);
-        if ((ret == -1) && (errno != ENOENT)) {
+        if ((ret < 0) && (-ret != ENOENT)) {
                 gf_log (this->name, GF_LOG_WARNING, "%s: lookup failed (%s)",
-                        loc->path, strerror (errno));
+                        loc->path, strerror (-ret));
+                ret = -1;
                 goto out;
         }
 
         /* we no more require this key */
-        dict_del (dict, DHT_LINKFILE_KEY);
+        dict_del (dict, conf->link_xattr_name);
 
         /* file exists in target node, only if it is 'linkfile' its valid,
            otherwise, error out */
         if (!ret) {
-                if (!check_is_linkfile (loc->inode, &stbuf, rsp_dict)) {
+                if (!check_is_linkfile (loc->inode, &stbuf, rsp_dict,
+                                        conf->link_xattr_name)) {
                         gf_log (this->name, GF_LOG_WARNING,
                                 "%s: file exists in destination", loc->path);
                         ret = -1;
@@ -559,7 +659,8 @@ migrate_special_files (xlator_t *this, xlator_t *from, xlator_t *to, loc_t *loc,
                 if (ret) {
                         gf_log (this->name, GF_LOG_WARNING,
                                 "%s: failed to delete the linkfile (%s)",
-                                loc->path, strerror (errno));
+                                loc->path, strerror (-ret));
+                        ret = -1;
                         goto out;
                 }
         }
@@ -579,15 +680,17 @@ migrate_special_files (xlator_t *this, xlator_t *from, xlator_t *to, loc_t *loc,
                 if (ret < 0) {
                         gf_log (this->name, GF_LOG_WARNING,
                                 "%s: readlink on symlink failed (%s)",
-                                loc->path, strerror (errno));
+                                loc->path, strerror (-ret));
+                        ret = -1;
                         goto out;
                 }
 
-                ret = syncop_symlink (to, loc, link, dict);
+                ret = syncop_symlink (to, loc, link, dict, 0);
                 if (ret) {
                         gf_log (this->name, GF_LOG_WARNING,
                                 "%s: creating symlink failed (%s)",
-                                loc->path, strerror (errno));
+                                loc->path, strerror (-ret));
+                        ret = -1;
                         goto out;
                 }
 
@@ -597,18 +700,31 @@ migrate_special_files (xlator_t *this, xlator_t *from, xlator_t *to, loc_t *loc,
         ret = syncop_mknod (to, loc, st_mode_from_ia (buf->ia_prot,
                                                       buf->ia_type),
                             makedev (ia_major (buf->ia_rdev),
-                                     ia_minor (buf->ia_rdev)), dict);
+                                     ia_minor (buf->ia_rdev)), dict, 0);
         if (ret) {
                 gf_log (this->name, GF_LOG_WARNING, "%s: mknod failed (%s)",
-                        loc->path, strerror (errno));
+                        loc->path, strerror (-ret));
+                ret = -1;
                 goto out;
         }
 
 done:
+        ret = syncop_setattr (to, loc, buf,
+                              (GF_SET_ATTR_UID | GF_SET_ATTR_GID |
+                               GF_SET_ATTR_MODE), NULL, NULL);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: failed to perform setattr on %s (%s)",
+                        loc->path, to->name, strerror (-ret));
+                ret = -1;
+        }
+
         ret = syncop_unlink (from, loc);
-        if (ret)
+        if (ret) {
                 gf_log (this->name, GF_LOG_WARNING, "%s: unlink failed (%s)",
-                        loc->path, strerror (errno));
+                        loc->path, strerror (-ret));
+                ret = -1;
+        }
 
 out:
         if (dict)
@@ -642,6 +758,7 @@ dht_migrate_file (xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
         dict_t         *xattr          = NULL;
         dict_t         *xattr_rsp      = NULL;
         int             file_has_holes = 0;
+        dht_conf_t     *conf           = this->private;
 
         gf_log (this->name, GF_LOG_INFO, "%s: attempting to move from %s to %s",
                 loc->path, from->name, to->name);
@@ -650,7 +767,7 @@ dht_migrate_file (xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
         if (!dict)
                 goto out;
 
-        ret = dict_set_int32 (dict, DHT_LINKFILE_KEY, 256);
+        ret = dict_set_int32 (dict, conf->link_xattr_name, 256);
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "%s: failed to set 'linkto' key in dict", loc->path);
@@ -661,21 +778,24 @@ dht_migrate_file (xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
         ret = syncop_lookup (from, loc, dict, &stbuf, &xattr_rsp, NULL);
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR, "%s: lookup failed on %s (%s)",
-                        loc->path, from->name, strerror (errno));
+                        loc->path, from->name, strerror (-ret));
+                ret = -1;
                 goto out;
         }
 
         /* we no more require this key */
-        dict_del (dict, DHT_LINKFILE_KEY);
+        dict_del (dict, conf->link_xattr_name);
 
         /* preserve source mode, so set the same to the destination */
         src_ia_prot = stbuf.ia_prot;
 
         /* Check if file can be migrated */
         ret = __is_file_migratable (this, loc, &stbuf, xattr_rsp, flag);
-        if (ret)
+        if (ret) {
+                if (ret == -2)
+                        ret = 0;
                 goto out;
-
+        }
         /* Take care of the special files */
         if (!IA_ISREG (stbuf.ia_type)) {
                 /* Special files */
@@ -683,9 +803,18 @@ dht_migrate_file (xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
                 goto out;
         }
 
+        /* TODO: move all xattr related operations to fd based operations */
+        ret = syncop_listxattr (from, loc, &xattr);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: failed to get xattr from %s (%s)",
+                        loc->path, from->name, strerror (-ret));
+                ret = -1;
+        }
+
         /* create the destination, with required modes/xattr */
         ret = __dht_rebalance_create_dst_file (to, from, loc, &stbuf,
-                                               dict, &dst_fd);
+                                               dict, &dst_fd, xattr);
         if (ret)
                 goto out;
 
@@ -702,10 +831,12 @@ dht_migrate_file (xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
                 goto out;
         }
 
+
         ret = syncop_fstat (from, src_fd, &stbuf);
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR, "failed to lookup %s on %s (%s)",
-                        loc->path, from->name, strerror (errno));
+                        loc->path, from->name, strerror (-ret));
+                ret = -1;
                 goto out;
         }
 
@@ -724,33 +855,22 @@ dht_migrate_file (xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
                 if (ret) {
                         gf_log (this->name, GF_LOG_ERROR,
                                 "%s: failed to reset target size back to 0 (%s)",
-                                loc->path, strerror (errno));
+                                loc->path, strerror (-ret));
                 }
 
                 ret = -1;
                 goto out;
         }
 
-        /* TODO: move all xattr related operations to fd based operations */
-        ret = syncop_listxattr (from, loc, &xattr);
-        if (ret == -1)
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to get xattr from %s (%s)",
-                        loc->path, from->name, strerror (errno));
-
-        ret = syncop_setxattr (to, loc, xattr, 0);
-        if (ret == -1)
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to set xattr on %s (%s)",
-                        loc->path, to->name, strerror (errno));
-
         /* TODO: Sync the locks */
 
-        ret = syncop_fsync (to, dst_fd);
-        if (ret)
+        ret = syncop_fsync (to, dst_fd, 0);
+        if (ret) {
                 gf_log (this->name, GF_LOG_WARNING,
                         "%s: failed to fsync on %s (%s)",
-                        loc->path, to->name, strerror (errno));
+                        loc->path, to->name, strerror (-ret));
+                ret = -1;
+        }
 
 
         /* Phase 2 - Data-Migration Complete, Housekeeping updates pending */
@@ -760,7 +880,8 @@ dht_migrate_file (xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
                 /* Failed to get the stat info */
                 gf_log (this->name, GF_LOG_ERROR,
                         "failed to fstat file %s on %s (%s)",
-                        loc->path, from->name, strerror (errno));
+                        loc->path, from->name, strerror (-ret));
+                ret = -1;
                 goto out;
         }
 
@@ -782,7 +903,8 @@ dht_migrate_file (xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
         if (ret) {
                 gf_log (this->name, GF_LOG_WARNING,
                         "%s: failed to perform setattr on %s (%s)",
-                        loc->path, to->name, strerror (errno));
+                        loc->path, to->name, strerror (-ret));
+                ret = -1;
                 goto out;
         }
 
@@ -793,7 +915,8 @@ dht_migrate_file (xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
         if (ret) {
                 gf_log (this->name, GF_LOG_WARNING,
                         "%s: failed to perform setattr on %s (%s)",
-                        loc->path, to->name, strerror (errno));
+                        loc->path, to->name, strerror (-ret));
+                ret = -1;
         }
 
         /* Make the source as a linkfile first before deleting it */
@@ -803,16 +926,37 @@ dht_migrate_file (xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
         if (ret) {
                 gf_log (this->name, GF_LOG_WARNING,             \
                         "%s: failed to perform setattr on %s (%s)",
-                        loc->path, from->name, strerror (errno));
+                        loc->path, from->name, strerror (-ret));
+                ret = -1;
                 goto out;
         }
 
+       /* Free up the data blocks on the source node, as the whole
+           file is migrated */
+        ret = syncop_ftruncate (from, src_fd, 0);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: failed to perform truncate on %s (%s)",
+                        loc->path, from->name, strerror (-ret));
+                ret = -1;
+        }
+
+        /* remove the 'linkto' xattr from the destination */
+        ret = syncop_fremovexattr (to, dst_fd, conf->link_xattr_name, 0);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: failed to perform removexattr on %s (%s)",
+                        loc->path, to->name, strerror (-ret));
+                ret = -1;
+        }
+
         /* Do a stat and check the gfid before unlink */
         ret = syncop_stat (from, loc, &empty_iatt);
         if (ret) {
                 gf_log (this->name, GF_LOG_WARNING,
                         "%s: failed to do a stat on %s (%s)",
-                        loc->path, from->name, strerror (errno));
+                        loc->path, from->name, strerror (-ret));
+                ret = -1;
                 goto out;
         }
 
@@ -822,33 +966,18 @@ dht_migrate_file (xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
                 if (ret) {
                         gf_log (this->name, GF_LOG_WARNING,
                                 "%s: failed to perform unlink on %s (%s)",
-                                loc->path, from->name, strerror (errno));
+                                loc->path, from->name, strerror (-ret));
+                        ret = -1;
                         goto out;
                 }
         }
 
-        /* Free up the data blocks on the source node, as the whole
-           file is migrated */
-        ret = syncop_ftruncate (from, src_fd, 0);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to perform truncate on %s (%s)",
-                        loc->path, from->name, strerror (errno));
-        }
-
-        /* remove the 'linkto' xattr from the destination */
-        ret = syncop_fremovexattr (to, dst_fd, DHT_LINKFILE_KEY);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to perform removexattr on %s (%s)",
-                        loc->path, to->name, strerror (errno));
-        }
-
         ret = syncop_lookup (this, loc, NULL, NULL, NULL, NULL);
         if (ret) {
                 gf_log (this->name, GF_LOG_DEBUG,
                         "%s: failed to lookup the file on subvolumes (%s)",
-                        loc->path, strerror (errno));
+                        loc->path, strerror (-ret));
+                ret = -1;
         }
 
         gf_log (this->name, GF_LOG_INFO,
@@ -954,14 +1083,13 @@ dht_start_rebalance_task (xlator_t *this, call_frame_t *frame)
 }
 
 int
-gf_listener_stop (void)
+gf_listener_stop (xlator_t *this)
 {
         glusterfs_ctx_t  *ctx = NULL;
         cmd_args_t       *cmd_args = NULL;
         int              ret = 0;
-        xlator_t         *this = NULL;
 
-        ctx = glusterfs_ctx_get ();
+        ctx = this->ctx;
         GF_ASSERT (ctx);
         cmd_args = &ctx->cmd_args;
         if (cmd_args->sock_file) {
@@ -972,7 +1100,6 @@ gf_listener_stop (void)
         }
 
         if (ret) {
-                this = THIS;
                 gf_log (this->name, GF_LOG_ERROR, "Failed to unlink listener "
                         "socket %s, error: %s", cmd_args->sock_file,
                         strerror (errno));
@@ -1013,10 +1140,10 @@ gf_defrag_handle_migrate_error (int32_t op_errno, gf_defrag_info_t *defrag)
 {
         /* if errno is not ENOSPC or ENOTCONN, we can still continue
            with rebalance process */
-        if ((errno != ENOSPC) || (errno != ENOTCONN))
+        if ((op_errno != ENOSPC) || (op_errno != ENOTCONN))
                 return 1;
 
-        if (errno == ENOTCONN) {
+        if (op_errno == ENOTCONN) {
                 /* Most probably mount point went missing (mostly due
                    to a brick down), say rebalance failure to user,
                    let him restart it if everything is fine */
@@ -1024,7 +1151,7 @@ gf_defrag_handle_migrate_error (int32_t op_errno, gf_defrag_info_t *defrag)
                 return -1;
         }
 
-        if (errno == ENOSPC) {
+        if (op_errno == ENOSPC) {
                 /* rebalance process itself failed, may be
                    remote brick went down, or write failed due to
                    disk full etc etc.. */
@@ -1035,6 +1162,31 @@ gf_defrag_handle_migrate_error (int32_t op_errno, gf_defrag_info_t *defrag)
         return 0;
 }
 
+static gf_boolean_t
+gf_defrag_pattern_match (gf_defrag_info_t *defrag, char *name, uint64_t size)
+{
+        gf_defrag_pattern_list_t *trav = NULL;
+        gf_boolean_t               match = _gf_false;
+        gf_boolean_t               ret = _gf_false;
+
+        GF_VALIDATE_OR_GOTO ("dht", defrag, out);
+
+        trav = defrag->defrag_pattern;
+        while (trav) {
+                if (!fnmatch (trav->path_pattern, name, FNM_NOESCAPE)) {
+                        match = _gf_true;
+                        break;
+                }
+                trav = trav->next;
+        }
+
+        if ((match == _gf_true) && (size >= trav->size))
+                ret = _gf_true;
+
+ out:
+        return ret;
+}
+
 /* We do a depth first traversal of directories. But before we move into
  * subdirs, we complete the data migration of those directories whose layouts
  * have been fixed
@@ -1057,10 +1209,17 @@ gf_defrag_migrate_data (xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
         int32_t                  op_errno       = 0;
         char                    *uuid_str       = NULL;
         uuid_t                   node_uuid      = {0,};
-        int                      readdir_operrno = 0;
+        struct timeval           dir_start      = {0,};
+        struct timeval           end            = {0,};
+        double                   elapsed        = {0,};
+        struct timeval           start          = {0,};
+        int32_t                  err            = 0;
+        int                      loglevel       = GF_LOG_TRACE;
 
         gf_log (this->name, GF_LOG_INFO, "migrate data called on %s",
                 loc->path);
+        gettimeofday (&dir_start, NULL);
+
         fd = fd_create (loc->inode, defrag->pid);
         if (!fd) {
                 gf_log (this->name, GF_LOG_ERROR, "Failed to create fd");
@@ -1071,6 +1230,7 @@ gf_defrag_migrate_data (xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR, "Failed to open dir %s",
                         loc->path);
+                ret = -1;
                 goto out;
         }
 
@@ -1078,17 +1238,21 @@ gf_defrag_migrate_data (xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
 
         while ((ret = syncop_readdirp (this, fd, 131072, offset, NULL,
                                        &entries)) != 0) {
-                if (ret < 0)
-                        break;
 
-                /* Need to keep track of ENOENT errno, that means, there is no
-                   need to send more readdirp() */
-                readdir_operrno = errno;
+                if (ret < 0) {
 
-                free_entries = _gf_true;
+                        gf_log (this->name, GF_LOG_ERROR, "Readdir returned %s."
+                                " Aborting migrate-data",
+                                strerror(-ret));
+                        ret = -1;
+                        goto out;
+                }
 
                 if (list_empty (&entries.list))
                         break;
+
+                free_entries = _gf_true;
+
                 list_for_each_entry_safe (entry, tmp, &entries.list, list) {
                         if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) {
                                 ret = 1;
@@ -1105,7 +1269,15 @@ gf_defrag_migrate_data (xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
                                 continue;
 
                         defrag->num_files_lookedup++;
-
+                        if (defrag->stats == _gf_true) {
+                                gettimeofday (&start, NULL);
+                        }
+                        if (defrag->defrag_pattern &&
+                            (gf_defrag_pattern_match (defrag, entry->d_name,
+                                                      entry->d_stat.ia_size)
+                             == _gf_false)) {
+                                continue;
+                        }
                         loc_wipe (&entry_loc);
                         ret =dht_build_child_loc (this, &entry_loc, loc,
                                                   entry->d_name);
@@ -1140,6 +1312,7 @@ gf_defrag_migrate_data (xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
                         if (ret) {
                                 gf_log (this->name, GF_LOG_ERROR, "%s"
                                         " lookup failed", entry_loc.path);
+                                ret = -1;
                                 continue;
                         }
 
@@ -1148,6 +1321,7 @@ gf_defrag_migrate_data (xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
                         if(ret < 0) {
                                 gf_log (this->name, GF_LOG_ERROR, "Failed to "
                                         "get node-uuid for %s", entry_loc.path);
+                                ret = -1;
                                 continue;
                         }
 
@@ -1157,6 +1331,7 @@ gf_defrag_migrate_data (xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
                                 gf_log (this->name, GF_LOG_ERROR, "Failed to "
                                         "get node-uuid from dict for %s",
                                         entry_loc.path);
+                                ret = -1;
                                 continue;
                         }
 
@@ -1181,30 +1356,50 @@ gf_defrag_migrate_data (xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
 
 
                         /* if distribute is present, it will honor this key.
-                         * -1 is returned if distribute is not present or file
-                         * doesn't have a link-file. If file has link-file, the
-                         * path of link-file will be the value, and also that
-                         * guarantees that file has to be mostly migrated */
+                         * -1, ENODATA is returned if distribute is not present
+                         * or file doesn't have a link-file. If file has
+                         * link-file, the path of link-file will be the value,
+                         * and also that guarantees that file has to be mostly
+                         * migrated */
 
                         ret = syncop_getxattr (this, &entry_loc, &dict,
                                                GF_XATTR_LINKINFO_KEY);
                         if (ret < 0) {
-                                gf_log (this->name, GF_LOG_TRACE, "failed to "
-                                        "get link-to key for %s",
-                                        entry_loc.path);
+                                if (-ret != ENODATA) {
+                                        loglevel = GF_LOG_ERROR;
+                                        defrag->total_failures += 1;
+                                } else {
+                                        loglevel = GF_LOG_TRACE;
+                                }
+                                gf_log (this->name, loglevel, "%s: failed to "
+                                        "get "GF_XATTR_LINKINFO_KEY" key - %s",
+                                        entry_loc.path, strerror (-ret));
+                                ret = -1;
                                 continue;
                         }
 
                         ret = syncop_setxattr (this, &entry_loc, migrate_data,
                                                0);
                         if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR, "migrate-data"
-                                        " failed for %s", entry_loc.path);
-                                defrag->total_failures +=1;
+                                err = op_errno;
+                                /* errno is overloaded. See
+                                 * rebalance_task_completion () */
+                                if (err != ENOSPC) {
+                                        gf_log (this->name, GF_LOG_DEBUG,
+                                                "migrate-data skipped for %s"
+                                                " due to space constraints",
+                                                entry_loc.path);
+                                        defrag->skipped +=1;
+                                } else{
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "migrate-data failed for %s",
+                                                entry_loc.path);
+                                        defrag->total_failures +=1;
+                                }
                         }
 
-                        if (ret == -1) {
-                                op_errno = errno;
+                        if (ret < 0) {
+                                op_errno = -ret;
                                 ret = gf_defrag_handle_migrate_error (op_errno,
                                                                       defrag);
 
@@ -1225,15 +1420,27 @@ gf_defrag_migrate_data (xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
                                 defrag->total_data += iatt.ia_size;
                         }
                         UNLOCK (&defrag->lock);
+                        if (defrag->stats == _gf_true) {
+                                gettimeofday (&end, NULL);
+                                elapsed = (end.tv_sec - start.tv_sec) * 1e6 +
+                                          (end.tv_usec - start.tv_usec);
+                                gf_log (this->name, GF_LOG_INFO, "Migration of "
+                                        "file:%s size:%"PRIu64" bytes took %.2f"
+                                        "secs", entry_loc.path, iatt.ia_size,
+                                         elapsed/1e6);
+                        }
                 }
 
                 gf_dirent_free (&entries);
                 free_entries = _gf_false;
                 INIT_LIST_HEAD (&entries.list);
-
-                if (readdir_operrno == ENOENT)
-                        break;
         }
+
+        gettimeofday (&end, NULL);
+        elapsed = (end.tv_sec - dir_start.tv_sec) * 1e6 +
+                  (end.tv_usec - dir_start.tv_usec);
+        gf_log (this->name, GF_LOG_INFO, "Migration operation on dir %s took "
+                "%.2f secs", loc->path, elapsed/1e6);
         ret = 0;
 out:
         if (free_entries)
@@ -1250,7 +1457,6 @@ out:
 
 }
 
-
 int
 gf_defrag_fix_layout (xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
                   dict_t *fix_layout, dict_t *migrate_data)
@@ -1270,6 +1476,7 @@ gf_defrag_fix_layout (xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR, "Lookup failed on %s",
                         loc->path);
+                ret = -1;
                 goto out;
         }
 
@@ -1300,12 +1507,19 @@ gf_defrag_fix_layout (xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
         while ((ret = syncop_readdirp (this, fd, 131072, offset, NULL,
                 &entries)) != 0)
         {
-                if ((ret < 0) || (ret && (errno == ENOENT)))
-                        break;
-                free_entries = _gf_true;
+
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR, "Readdir returned %s"
+                                ". Aborting fix-layout",strerror(-ret));
+                        ret = -1;
+                        goto out;
+                }
 
                 if (list_empty (&entries.list))
                         break;
+
+                free_entries = _gf_true;
+
                 list_for_each_entry_safe (entry, tmp, &entries.list, list) {
                         if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) {
                                 ret = 1;
@@ -1332,7 +1546,7 @@ gf_defrag_fix_layout (xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
 
                         if (uuid_is_null (entry->d_stat.ia_gfid)) {
                                 gf_log (this->name, GF_LOG_ERROR, "%s/%s"
-                                        "gfid not present", loc->path,
+                                        " gfid not present", loc->path,
                                          entry->d_name);
                                 continue;
                         }
@@ -1342,7 +1556,7 @@ gf_defrag_fix_layout (xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
                         uuid_copy (entry_loc.gfid, entry->d_stat.ia_gfid);
                         if (uuid_is_null (loc->gfid)) {
                                 gf_log (this->name, GF_LOG_ERROR, "%s/%s"
-                                        "gfid not present", loc->path,
+                                        " gfid not present", loc->path,
                                          entry->d_name);
                                 continue;
                         }
@@ -1354,6 +1568,7 @@ gf_defrag_fix_layout (xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
                         if (ret) {
                                 gf_log (this->name, GF_LOG_ERROR, "%s"
                                         " lookup failed", entry_loc.path);
+                                ret = -1;
                                 continue;
                         }
 
@@ -1364,6 +1579,8 @@ gf_defrag_fix_layout (xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
                                         "failed for %s", entry_loc.path);
                                 defrag->defrag_status =
                                 GF_DEFRAG_STATUS_FAILED;
+                                defrag->total_failures ++;
+                                ret = -1;
                                 goto out;
                         }
                         ret = gf_defrag_fix_layout (this, defrag, &entry_loc,
@@ -1372,6 +1589,7 @@ gf_defrag_fix_layout (xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
                         if (ret) {
                                 gf_log (this->name, GF_LOG_ERROR, "Fix layout "
                                         "failed for %s", entry_loc.path);
+                                defrag->total_failures++;
                                 goto out;
                         }
 
@@ -1418,7 +1636,7 @@ gf_defrag_start_crawl (void *data)
         if (!this)
                 goto out;
 
-        ctx = glusterfs_ctx_get ();
+        ctx = this->ctx;
         if (!ctx)
                 goto out;
 
@@ -1429,6 +1647,8 @@ gf_defrag_start_crawl (void *data)
         defrag = conf->defrag;
         if (!defrag)
                 goto out;
+
+        gettimeofday (&defrag->start_time, NULL);
         dht_build_root_inode (this, &defrag->root_inode);
         if (!defrag->root_inode)
                 goto out;
@@ -1441,6 +1661,7 @@ gf_defrag_start_crawl (void *data)
 
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR, "look up on / failed");
+                ret = -1;
                 goto out;
         }
 
@@ -1460,6 +1681,8 @@ gf_defrag_start_crawl (void *data)
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR, "fix layout on %s failed",
                         loc.path);
+                defrag->total_failures++;
+                ret = -1;
                 goto out;
         }
 
@@ -1513,7 +1736,7 @@ out:
 static int
 gf_defrag_done  (int ret, call_frame_t *sync_frame, void *data)
 {
-        gf_listener_stop();
+        gf_listener_stop (sync_frame->this);
 
         STACK_DESTROY (sync_frame->root);
         kill (getpid(), SIGTERM);
@@ -1566,7 +1789,11 @@ gf_defrag_status_get (gf_defrag_info_t *defrag, dict_t *dict)
         uint64_t size   = 0;
         uint64_t lookup = 0;
         uint64_t failures = 0;
+        uint64_t skipped = 0;
         char     *status = "";
+        double   elapsed = 0;
+        struct timeval end = {0,};
+
 
         if (!defrag)
                 goto out;
@@ -1579,6 +1806,11 @@ gf_defrag_status_get (gf_defrag_info_t *defrag, dict_t *dict)
         size   = defrag->total_data;
         lookup = defrag->num_files_lookedup;
         failures = defrag->total_failures;
+        skipped = defrag->skipped;
+
+        gettimeofday (&end, NULL);
+
+        elapsed = end.tv_sec - defrag->start_time.tv_sec;
 
         if (!dict)
                 goto log;
@@ -1598,12 +1830,27 @@ gf_defrag_status_get (gf_defrag_info_t *defrag, dict_t *dict)
                 gf_log (THIS->name, GF_LOG_WARNING,
                         "failed to set lookedup file count");
 
+
         ret = dict_set_int32 (dict, "status", defrag->defrag_status);
         if (ret)
                 gf_log (THIS->name, GF_LOG_WARNING,
                         "failed to set status");
+        if (elapsed) {
+                ret = dict_set_double (dict, "run-time", elapsed);
+                if (ret)
+                        gf_log (THIS->name, GF_LOG_WARNING,
+                                "failed to set run-time");
+        }
 
         ret = dict_set_uint64 (dict, "failures", failures);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_WARNING,
+                        "failed to set failure count");
+
+        ret = dict_set_uint64 (dict, "skipped", skipped);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_WARNING,
+                        "failed to set skipped file count");
 log:
         switch (defrag->defrag_status) {
         case GF_DEFRAG_STATUS_NOT_STARTED:
@@ -1621,12 +1868,15 @@ log:
         case GF_DEFRAG_STATUS_FAILED:
                 status = "failed";
                 break;
+        default:
+                break;
         }
 
-        gf_log (THIS->name, GF_LOG_INFO, "Rebalance is %s", status);
+        gf_log (THIS->name, GF_LOG_INFO, "Rebalance is %s. Time taken is %.2f "
+                "secs", status, elapsed);
         gf_log (THIS->name, GF_LOG_INFO, "Files migrated: %"PRIu64", size: %"
-                PRIu64", lookups: %"PRIu64", failures: %"PRIu64, files, size,
-                lookup, failures);
+                PRIu64", lookups: %"PRIu64", failures: %"PRIu64", skipped: "
+                "%"PRIu64, files, size, lookup, failures, skipped);
 
 
 out:
@@ -1634,7 +1884,8 @@ out:
 }
 
 int
-gf_defrag_stop (gf_defrag_info_t *defrag, dict_t *output)
+gf_defrag_stop (gf_defrag_info_t *defrag, gf_defrag_status_t status,
+                dict_t *output)
 {
         /* TODO: set a variable 'stop_defrag' here, it should be checked
            in defrag loop */
@@ -1646,7 +1897,7 @@ gf_defrag_stop (gf_defrag_info_t *defrag, dict_t *output)
         }
 
         gf_log ("", GF_LOG_INFO, "Received stop command on rebalance");
-        defrag->defrag_status = GF_DEFRAG_STATUS_STOPPED;
+        defrag->defrag_status = status;
 
         if (output)
                 gf_defrag_status_get (defrag, output);
diff --git a/xlators/cluster/dht/src/dht-rename.c b/xlators/cluster/dht/src/dht-rename.c
index f0eecb2d7..925538cc8 100644
--- a/xlators/cluster/dht/src/dht-rename.c
+++ b/xlators/cluster/dht/src/dht-rename.c
@@ -306,6 +306,34 @@ err:
                           NULL, NULL);
         return 0;
 }
+#define DHT_MARK_FOP_INTERNAL(xattr) do {                                      \
+                int tmp = -1;                                                  \
+                if (!xattr) {                                                  \
+                        xattr = dict_new ();                                   \
+                        if (!xattr)                                            \
+                                break;                                         \
+                }                                                              \
+                tmp = dict_set_str (xattr, GLUSTERFS_INTERNAL_FOP_KEY, "yes"); \
+                if (tmp) {                                                     \
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to set"      \
+                                " internal dict key for %s", local->loc.path); \
+                }                                                              \
+        }while (0)
+
+#define DHT_MARKER_DONT_ACCOUNT(xattr) do {                             \
+                int tmp = -1;                                                  \
+                if (!xattr) {                                                  \
+                        xattr = dict_new ();                                   \
+                        if (!xattr)                                            \
+                                break;                                         \
+                }                                                              \
+                tmp = dict_set_str (xattr, GLUSTERFS_MARKER_DONT_ACCOUNT_KEY,  \
+                                    "yes");                                    \
+                if (tmp) {                                                     \
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to set"      \
+                                " marker dont account key for %s", local->loc.path); \
+                }                                                              \
+        }while (0)
 
 int
 dht_rename_done (call_frame_t *frame, xlator_t *this)
@@ -377,7 +405,7 @@ dht_rename_cleanup (call_frame_t *frame)
         xlator_t    *dst_hashed = NULL;
         xlator_t    *dst_cached = NULL;
         int          call_cnt = 0;
-
+        dict_t      *xattr      = NULL;
 
         local = frame->local;
         this  = frame->this;
@@ -401,24 +429,53 @@ dht_rename_cleanup (call_frame_t *frame)
         if (!call_cnt)
                 goto nolinks;
 
+        DHT_MARK_FOP_INTERNAL (xattr);
+
         if (dst_hashed != src_hashed && dst_hashed != src_cached) {
+                dict_t *xattr_new = NULL;
+
                 gf_log (this->name, GF_LOG_TRACE,
                         "unlinking linkfile %s @ %s => %s",
                         local->loc.path, dst_hashed->name, src_cached->name);
+
+                xattr_new = dict_copy_with_ref (xattr, NULL);
+
+
+                DHT_MARKER_DONT_ACCOUNT(xattr_new);
+
                 STACK_WIND (frame, dht_rename_unlink_cbk,
                             dst_hashed, dst_hashed->fops->unlink,
-                            &local->loc, 0, NULL);
+                            &local->loc, 0, xattr_new);
+
+                dict_unref (xattr_new);
+                xattr_new = NULL;
         }
 
         if (src_cached != dst_hashed) {
+                dict_t *xattr_new = NULL;
+
                 gf_log (this->name, GF_LOG_TRACE,
                         "unlinking link %s => %s (%s)", local->loc.path,
                         local->loc2.path, src_cached->name);
+
+                xattr_new = dict_copy_with_ref (xattr, NULL);
+
+                if (uuid_compare (local->loc.pargfid,
+                                  local->loc2.pargfid) == 0) {
+                        DHT_MARKER_DONT_ACCOUNT(xattr_new);
+                }
+
                 STACK_WIND (frame, dht_rename_unlink_cbk,
                             src_cached, src_cached->fops->unlink,
-                            &local->loc2, 0, NULL);
+                            &local->loc2, 0, xattr_new);
+
+                dict_unref (xattr_new);
+                xattr_new = NULL;
         }
 
+        if (xattr)
+                dict_unref (xattr);
+
         return 0;
 
 nolinks:
@@ -482,6 +539,7 @@ dht_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         xlator_t     *rename_subvol = NULL;
         call_frame_t *link_frame = NULL;
         dht_local_t *link_local = NULL;
+        dict_t       *xattr     = NULL;
 
         local = frame->local;
         prev = cookie;
@@ -522,7 +580,8 @@ dht_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 uuid_copy (link_local->gfid, local->loc.inode->gfid);
 
                 dht_linkfile_create (link_frame, dht_rename_links_create_cbk,
-                                     src_cached, dst_hashed, &link_local->loc);
+                                     this, src_cached, dst_hashed,
+                                     &link_local->loc);
         }
 
 err:
@@ -564,24 +623,47 @@ err:
         if (local->call_cnt == 0)
                 goto unwind;
 
+        DHT_MARK_FOP_INTERNAL (xattr);
+
         if (src_cached != dst_hashed && src_cached != dst_cached) {
+                dict_t *xattr_new = NULL;
+
+                xattr_new = dict_copy_with_ref (xattr, NULL);
+
                 gf_log (this->name, GF_LOG_TRACE,
                         "deleting old src datafile %s @ %s",
                         local->loc.path, src_cached->name);
 
+                if (uuid_compare (local->loc.pargfid,
+                                  local->loc2.pargfid) == 0) {
+                        DHT_MARKER_DONT_ACCOUNT(xattr_new);
+                }
+
                 STACK_WIND (frame, dht_rename_unlink_cbk,
                             src_cached, src_cached->fops->unlink,
-                            &local->loc, 0, NULL);
+                            &local->loc, 0, xattr_new);
+
+                dict_unref (xattr_new);
+                xattr_new = NULL;
         }
 
         if (src_hashed != rename_subvol && src_hashed != src_cached) {
+                dict_t *xattr_new = NULL;
+
+                xattr_new = dict_copy_with_ref (xattr, NULL);
+
                 gf_log (this->name, GF_LOG_TRACE,
                         "deleting old src linkfile %s @ %s",
                         local->loc.path, src_hashed->name);
 
+                DHT_MARKER_DONT_ACCOUNT(xattr_new);
+
                 STACK_WIND (frame, dht_rename_unlink_cbk,
                             src_hashed, src_hashed->fops->unlink,
-                            &local->loc, 0, NULL);
+                            &local->loc, 0, xattr_new);
+
+                dict_unref (xattr_new);
+                xattr_new = NULL;
         }
 
         if (dst_cached
@@ -593,8 +675,10 @@ err:
 
                 STACK_WIND (frame, dht_rename_unlink_cbk,
                             dst_cached, dst_cached->fops->unlink,
-                            &local->loc2, 0, NULL);
+                            &local->loc2, 0, xattr);
         }
+        if (xattr)
+                dict_unref (xattr);
         return 0;
 
 unwind:
@@ -602,12 +686,16 @@ unwind:
         WIPE (&local->postoldparent);
         WIPE (&local->preparent);
         WIPE (&local->postparent);
+        if (xattr)
+                dict_unref (xattr);
 
         dht_rename_done (frame, this);
 
         return 0;
 
 cleanup:
+        if (xattr)
+                dict_unref (xattr);
         dht_rename_cleanup (frame);
 
         return 0;
@@ -617,12 +705,13 @@ cleanup:
 int
 dht_do_rename (call_frame_t *frame)
 {
-        dht_local_t *local = NULL;
-        xlator_t    *dst_hashed = NULL;
-        xlator_t    *src_cached = NULL;
-        xlator_t    *dst_cached = NULL;
-        xlator_t    *this = NULL;
+        dht_local_t *local         = NULL;
+        xlator_t    *dst_hashed    = NULL;
+        xlator_t    *src_cached    = NULL;
+        xlator_t    *dst_cached    = NULL;
+        xlator_t    *this          = NULL;
         xlator_t    *rename_subvol = NULL;
+        dict_t      *dict          = NULL;
 
 
         local = frame->local;
@@ -637,6 +726,10 @@ dht_do_rename (call_frame_t *frame)
         else
                 rename_subvol = dst_hashed;
 
+        if ((src_cached != dst_hashed) && (rename_subvol == dst_hashed)) {
+                DHT_MARKER_DONT_ACCOUNT(dict);
+        }
+
         gf_log (this->name, GF_LOG_TRACE,
                 "renaming %s => %s (%s)",
                 local->loc.path, local->loc2.path, rename_subvol->name);
@@ -645,7 +738,7 @@ dht_do_rename (call_frame_t *frame)
                 FRAME_SU_DO (frame, dht_local_t);
         STACK_WIND (frame, dht_rename_cbk,
                     rename_subvol, rename_subvol->fops->rename,
-                    &local->loc, &local->loc2, NULL);
+                    &local->loc, &local->loc2, dict);
 
         return 0;
 }
@@ -741,6 +834,7 @@ dht_rename_create_links (call_frame_t *frame)
         xlator_t    *dst_hashed = NULL;
         xlator_t    *dst_cached = NULL;
         int          call_cnt = 0;
+        dict_t      *xattr = NULL;
 
 
         local = frame->local;
@@ -751,18 +845,27 @@ dht_rename_create_links (call_frame_t *frame)
         dst_hashed = local->dst_hashed;
         dst_cached = local->dst_cached;
 
+        DHT_MARK_FOP_INTERNAL (xattr);
 
         if (src_cached == dst_cached) {
+                dict_t *xattr_new = NULL;
+
                 if (dst_hashed == dst_cached)
                         goto nolinks;
 
+                xattr_new = dict_copy_with_ref (xattr, NULL);
+
 		gf_log (this->name, GF_LOG_TRACE,
 			"unlinking dst linkfile %s @ %s",
 			local->loc2.path, dst_hashed->name);
 
+                DHT_MARKER_DONT_ACCOUNT(xattr_new);
+
 		STACK_WIND (frame, dht_rename_unlink_links_cbk,
 			    dst_hashed, dst_hashed->fops->unlink,
-			    &local->loc2, 0, NULL);
+			    &local->loc2, 0, xattr_new);
+
+                dict_unref (xattr_new);
                 return 0;
         }
 
@@ -779,17 +882,28 @@ dht_rename_create_links (call_frame_t *frame)
                         "linkfile %s @ %s => %s",
                         local->loc.path, dst_hashed->name, src_cached->name);
                 memcpy (local->gfid, local->loc.inode->gfid, 16);
-		dht_linkfile_create (frame, dht_rename_links_cbk,
+		dht_linkfile_create (frame, dht_rename_links_cbk, this,
 				     src_cached, dst_hashed, &local->loc);
 	}
 
 	if (src_cached != dst_hashed) {
+                dict_t *xattr_new = NULL;
+
+                xattr_new = dict_copy_with_ref (xattr, NULL);
+
 		gf_log (this->name, GF_LOG_TRACE,
 			"link %s => %s (%s)", local->loc.path,
 			local->loc2.path, src_cached->name);
+                if (uuid_compare (local->loc.pargfid,
+                                  local->loc2.pargfid) == 0) {
+                        DHT_MARKER_DONT_ACCOUNT(xattr_new);
+                }
+
 		STACK_WIND (frame, dht_rename_links_cbk,
 			    src_cached, src_cached->fops->link,
-			    &local->loc, &local->loc2, NULL);
+			    &local->loc, &local->loc2, xattr_new);
+
+                dict_unref (xattr_new);
 	}
 
 nolinks:
@@ -797,6 +911,8 @@ nolinks:
                 /* skip to next step */
                 dht_do_rename (frame);
         }
+        if (xattr)
+                dict_unref (xattr);
 
         return 0;
 }
diff --git a/xlators/cluster/dht/src/dht-selfheal.c b/xlators/cluster/dht/src/dht-selfheal.c
index 68d9416fa..0e6527544 100644
--- a/xlators/cluster/dht/src/dht-selfheal.c
+++ b/xlators/cluster/dht/src/dht-selfheal.c
@@ -17,6 +17,7 @@
 #include "glusterfs.h"
 #include "xlator.h"
 #include "dht-common.h"
+#include "glusterfs-acl.h"
 
 #define DHT_SET_LAYOUT_RANGE(layout,i,srt,chunk,cnt,path)    do {       \
                 layout->list[i].start = srt;                            \
@@ -28,42 +29,40 @@
                         layout->list[i].xlator->name, path);            \
         } while (0)
 
-static inline uint32_t
-dht_find_overlap (int idx, int cnk_idx, uint32_t start, uint32_t stop,
-                  uint32_t chunk_size)
-{
-        uint32_t overlap = 0;
-        uint32_t chunk_begin = 0;
+#define DHT_RESET_LAYOUT_RANGE(layout)    do {                          \
+                int cnt = 0;                                            \
+                for (cnt = 0; cnt < layout->cnt; cnt++ ) {              \
+                        layout->list[cnt].start = 0;                    \
+                        layout->list[cnt].stop  = 0;                    \
+                }                                                       \
+        } while (0)
 
-        chunk_begin = cnk_idx * chunk_size;
+static uint32_t
+dht_overlap_calc (dht_layout_t *old, int o, dht_layout_t *new, int n)
+{
+	if (o >= old->cnt || n >= new->cnt)
+		return 0;
 
-        /* There is no chance of overlap */
-        if ((chunk_begin > stop) ||
-            ((chunk_begin + chunk_size) < start))
-                goto out;
+	if (old->list[o].err > 0 || new->list[n].err > 0)
+		return 0;
 
-        if ((chunk_begin <= start) &&
-            ((chunk_begin + chunk_size) <= stop)) {
-                overlap = ((chunk_begin + chunk_size) - start);
-                goto out;
+        if (old->list[o].start == old->list[o].stop) {
+                return 0;
         }
 
-        if ((chunk_begin <= start) &&
-            ((chunk_begin + chunk_size) >= stop)) {
-                overlap = (stop - start);
-                goto out;
+        if (new->list[n].start == new->list[n].stop) {
+                return 0;
         }
 
-        if ((chunk_begin < stop) &&
-            ((chunk_begin + chunk_size) >= stop)) {
-                overlap = (stop - chunk_begin);
-                goto out;
-        }
+	if ((old->list[o].start > new->list[n].stop) ||
+	    (old->list[o].stop < new->list[n].start))
+		return 0;
 
-out:
-        return overlap;
+	return min (old->list[o].stop, new->list[n].stop) -
+	        max (old->list[o].start, new->list[n].start) + 1;
 }
 
+
 int
 dht_selfheal_dir_finish (call_frame_t *frame, xlator_t *this, int ret)
 {
@@ -118,7 +117,8 @@ dht_selfheal_dir_xattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 int
 dht_selfheal_dir_xattr_persubvol (call_frame_t *frame, loc_t *loc,
-                                  dht_layout_t *layout, int i)
+                                  dht_layout_t *layout, int i,
+                                  xlator_t *req_subvol)
 {
         xlator_t          *subvol = NULL;
         dict_t            *xattr = NULL;
@@ -126,16 +126,23 @@ dht_selfheal_dir_xattr_persubvol (call_frame_t *frame, loc_t *loc,
         xlator_t          *this = NULL;
         int32_t           *disk_layout = NULL;
         dht_local_t       *local = NULL;
-
+        dht_conf_t        *conf = NULL;
+        data_t            *data = NULL;
 
         local = frame->local;
-        subvol = layout->list[i].xlator;
+        if (req_subvol)
+                subvol = req_subvol;
+        else
+                subvol = layout->list[i].xlator;
         this = frame->this;
 
         GF_VALIDATE_OR_GOTO ("", this, err);
         GF_VALIDATE_OR_GOTO (this->name, layout, err);
         GF_VALIDATE_OR_GOTO (this->name, local, err);
         GF_VALIDATE_OR_GOTO (this->name, subvol, err);
+        VALIDATE_OR_GOTO (this->private, err);
+
+        conf = this->private;
 
         xattr = get_new_dict ();
         if (!xattr) {
@@ -150,8 +157,7 @@ dht_selfheal_dir_xattr_persubvol (call_frame_t *frame, loc_t *loc,
                 goto err;
         }
 
-        ret = dict_set_bin (xattr, "trusted.glusterfs.dht",
-                            disk_layout, 4 * 4);
+        ret = dict_set_bin (xattr, conf->xattr_name, disk_layout, 4 * 4);
         if (ret == -1) {
                 gf_log (this->name, GF_LOG_WARNING,
                         "%s: (subvol %s) failed to set xattr dictionary",
@@ -166,7 +172,16 @@ dht_selfheal_dir_xattr_persubvol (call_frame_t *frame, loc_t *loc,
                 layout->type, subvol->name, loc->path);
 
         dict_ref (xattr);
-
+        if (local->xattr) {
+                data = dict_get (local->xattr, QUOTA_LIMIT_KEY);
+                if (data) {
+                        ret = dict_add (xattr, QUOTA_LIMIT_KEY, data);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                        "set quota limit key on %s",loc->path);
+                        }
+                }
+        }
         if (!uuid_is_null (local->gfid))
                 uuid_copy (loc->gfid, local->gfid);
 
@@ -182,8 +197,7 @@ err:
         if (xattr)
                 dict_destroy (xattr);
 
-        if (disk_layout)
-                GF_FREE (disk_layout);
+        GF_FREE (disk_layout);
 
         dht_selfheal_dir_xattr_cbk (frame, subvol, frame->this,
                                     -1, ENOMEM, NULL);
@@ -197,21 +211,42 @@ dht_fix_dir_xattr (call_frame_t *frame, loc_t *loc, dht_layout_t *layout)
         int          i = 0;
         int          count = 0;
         xlator_t    *this = NULL;
+        dht_conf_t  *conf = NULL;
+        dht_layout_t *dummy = NULL;
 
         local = frame->local;
         this = frame->this;
+        conf = this->private;
 
         gf_log (this->name, GF_LOG_DEBUG,
                 "writing the new range for all subvolumes");
 
-        local->call_cnt = count = layout->cnt;
+        local->call_cnt = count = conf->subvolume_cnt;
 
         for (i = 0; i < layout->cnt; i++) {
-                dht_selfheal_dir_xattr_persubvol (frame, loc, layout, i);
+                dht_selfheal_dir_xattr_persubvol (frame, loc, layout, i, NULL);
 
                 if (--count == 0)
-                        break;
+                        goto out;
         }
+        /* if we are here, subvolcount > layout_count. subvols-per-directory
+         * option might be set here. We need to clear out layout from the
+         * non-participating subvolumes, else it will result in overlaps */
+        dummy = dht_layout_new (this, 1);
+        if (!dummy)
+                goto out;
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (_gf_false ==
+                    dht_is_subvol_in_layout (layout, conf->subvolumes[i])) {
+                        dht_selfheal_dir_xattr_persubvol (frame, loc, dummy, 0,
+                                                          conf->subvolumes[i]);
+                        if (--count == 0)
+                                break;
+                }
+        }
+
+        dht_layout_unref (this, dummy);
+out:
         return 0;
 }
 
@@ -222,9 +257,12 @@ dht_selfheal_dir_xattr (call_frame_t *frame, loc_t *loc, dht_layout_t *layout)
         int          missing_xattr = 0;
         int          i = 0;
         xlator_t    *this = NULL;
+        dht_conf_t   *conf = NULL;
+        dht_layout_t *dummy = NULL;
 
         local = frame->local;
         this = frame->this;
+        conf = this->private;
 
         for (i = 0; i < layout->cnt; i++) {
                 if (layout->list[i].err != -1 || !layout->list[i].stop) {
@@ -237,7 +275,14 @@ dht_selfheal_dir_xattr (call_frame_t *frame, loc_t *loc, dht_layout_t *layout)
                 }
                 missing_xattr++;
         }
-
+        /* Also account for subvolumes with no-layout. Used for zero'ing out
+         * the layouts and for setting quota key's if present */
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (_gf_false ==
+                    dht_is_subvol_in_layout (layout, conf->subvolumes[i])) {
+                        missing_xattr++;
+                }
+        }
         gf_log (this->name, GF_LOG_TRACE,
                 "%d subvolumes missing xattr for %s",
                 missing_xattr, loc->path);
@@ -248,16 +293,29 @@ dht_selfheal_dir_xattr (call_frame_t *frame, loc_t *loc, dht_layout_t *layout)
         }
 
         local->call_cnt = missing_xattr;
-
         for (i = 0; i < layout->cnt; i++) {
                 if (layout->list[i].err != -1 || !layout->list[i].stop)
                         continue;
 
-                dht_selfheal_dir_xattr_persubvol (frame, loc, layout, i);
+                dht_selfheal_dir_xattr_persubvol (frame, loc, layout, i, NULL);
 
                 if (--missing_xattr == 0)
                         break;
         }
+        dummy = dht_layout_new (this, 1);
+        if (!dummy)
+                goto out;
+        for (i = 0; i < conf->subvolume_cnt && missing_xattr; i++) {
+                if (_gf_false ==
+                    dht_is_subvol_in_layout (layout, conf->subvolumes[i])) {
+                        dht_selfheal_dir_xattr_persubvol (frame, loc, dummy, 0,
+                                                          conf->subvolumes[i]);
+                        missing_xattr--;
+                }
+        }
+
+        dht_layout_unref (this, dummy);
+out:
         return 0;
 }
 
@@ -494,7 +552,7 @@ dht_selfheal_layout_alloc_start (xlator_t *this, loc_t *loc,
         uint32_t      hashval = 0;
         int           ret = 0;
 
-        ret = dht_hash_compute (layout->type, loc->path, &hashval);
+        ret = dht_hash_compute (this, layout->type, loc->path, &hashval);
         if (ret == 0) {
                 start = (hashval % layout->cnt);
         }
@@ -517,7 +575,7 @@ dht_get_layout_count (xlator_t *this, dht_layout_t *layout, int new_layout)
                 for (j = 0; j < conf->subvolume_cnt; j++) {
                         if (conf->decommissioned_bricks[j] &&
                             conf->decommissioned_bricks[j] == layout->list[i].xlator) {
-                                layout->list[i].err = -EINVAL;
+                                layout->list[i].err = EINVAL;
                                 break;
                         }
                 }
@@ -525,9 +583,33 @@ dht_get_layout_count (xlator_t *this, dht_layout_t *layout, int new_layout)
 
         for (i = 0; i < layout->cnt; i++) {
                 err = layout->list[i].err;
-                if (err == -1 || err == 0) {
-                        layout->list[i].err = -1;
+                if (err == -1 || err == 0 || err == ENOENT) {
+			/* Setting list[i].err = -1 is an indication for
+			   dht_selfheal_layout_new_directory() to assign
+			   a range. We set it to -1 based on any one of
+			   the three criteria:
+
+			   - err == -1 already, which means directory
+			     existed but layout was not set on it.
+
+			   - err == 0, which means directory exists and
+			     has an old layout piece which will be
+			     overwritten now.
+
+			   - err == ENOENT, which means directory does
+			     not exist (possibly racing with mkdir or
+			     finishing half done mkdir). The missing
+			     directory will be attempted to be recreated.
+
+			     It is important to note that it is safe
+			     to race with mkdir() as self-heal and
+			     mkdir are idempotent operations. Both will
+			     strive to set the directory and layouts to
+			     the same final state.
+			*/
                         count++;
+			if (!err)
+				layout->list[i].err = -1;
                 }
         }
 
@@ -542,49 +624,126 @@ dht_get_layout_count (xlator_t *this, dht_layout_t *layout, int new_layout)
                 }
         }
 
-        count = ((layout->spread_cnt) ? layout->spread_cnt :
-                 ((count) ? count : 1));
+        /* if layout->spread_cnt is set, check if it is <= available
+         * subvolumes (down brick and decommissioned bricks are considered
+         * un-availbale). Else return count (available up bricks) */
+        count = ((layout->spread_cnt &&
+                 (layout->spread_cnt <= count)) ?
+                 layout->spread_cnt : ((count) ? count : 1));
 
         return count;
 }
 
 
+void dht_selfheal_layout_new_directory (call_frame_t *frame, loc_t *loc,
+					dht_layout_t *new_layout);
+
+void dht_layout_entry_swap (dht_layout_t *layout, int i, int j);
+void dht_layout_range_swap (dht_layout_t *layout, int i, int j);
+
+/*
+ * It's a bit icky using local variables in a macro, but it makes the rest
+ * of the code a lot clearer.
+ */
+#define OV_ENTRY(x,y)      table[x*new->cnt+y]
+
+void
+dht_selfheal_layout_maximize_overlap (call_frame_t *frame, loc_t *loc,
+				      dht_layout_t *new, dht_layout_t *old)
+{
+	int           i            = 0;
+	int           j            = 0;
+	uint32_t      curr_overlap = 0;
+	uint32_t      max_overlap  = 0;
+	int           max_overlap_idx = -1;
+	uint32_t      overlap      = 0;
+        uint32_t     *table = NULL;
+
+	dht_layout_sort_volname (old);
+	/* Now both old_layout->list[] and new_layout->list[]
+	   are match the same xlators/subvolumes. i.e,
+	   old_layout->[i] and new_layout->[i] are referring
+	   to the same subvolumes
+	*/
+
+        /* Build a table of overlaps between new[i] and old[j]. */
+        table = alloca(sizeof(overlap)*old->cnt*new->cnt);
+        if (!table) {
+                return;
+        }
+        memset(table,0,sizeof(overlap)*old->cnt*new->cnt);
+        for (i = 0; i < new->cnt; ++i) {
+                for (j = 0; j < old->cnt; ++j) {
+                        OV_ENTRY(i,j) = dht_overlap_calc(old,j,new,i);
+                }
+        }
+
+	for (i = 0; i < new->cnt; i++) {
+		if (new->list[i].err > 0) {
+			/* Subvol might be marked for decommission
+			   with EINVAL, or some other serious error
+			   marked with positive errno.
+			*/
+			continue;
+                }
+
+                max_overlap = 0;
+                max_overlap_idx = i;
+                for (j = (i + 1); j < new->cnt; ++j) {
+                        if (new->list[j].err > 0) {
+			        /* Subvol might be marked for decommission
+			        with EINVAL, or some other serious error
+			        marked with positive errno.
+			        */
+			        continue;
+                        }
+                        /* Calculate the overlap now. */
+                        curr_overlap = OV_ENTRY(i,i) + OV_ENTRY(j,j);
+                        /* Calculate the overlap after the proposed swap. */
+                        overlap = OV_ENTRY(i,j) + OV_ENTRY(j,i);
+                        /* Are we better than status quo? */
+                        if (overlap > curr_overlap) {
+                                overlap -= curr_overlap;
+                                /* Are we better than the previous choice? */
+                                if (overlap > max_overlap) {
+                                        max_overlap = overlap;
+                                        max_overlap_idx = j;
+                                }
+                        }
+                }
+
+		if (max_overlap_idx != i) {
+ 			dht_layout_range_swap (new, i, max_overlap_idx);
+                        /* Need to swap the table values too. */
+                        for (j = 0; j < old->cnt; ++j) {
+                                overlap = OV_ENTRY(i,j);
+                                OV_ENTRY(i,j) = OV_ENTRY(max_overlap_idx,j);
+                                OV_ENTRY(max_overlap_idx,j) = overlap;
+                        }
+                }
+	}
+}
+
+
 dht_layout_t *
 dht_fix_layout_of_directory (call_frame_t *frame, loc_t *loc,
                              dht_layout_t *layout)
 {
-        uint32_t      chunk        = 0;
-        uint32_t      start        = 0;
-        uint32_t      stop         = 0;
-        uint32_t      overlap      = 0;
-        uint32_t      max_overlap  = 0;
-        uint32_t      chunk_begin  = 0;
-        int           count        = 0;
-        int           cnt          = 0;
         int           i            = 0;
-        int           j            = 0;
-        int           k            = 0;
-        int           loop_cnt     = 0;
-        int           start_subvol = 0;
-        int          *fix_array    = NULL;
         xlator_t     *this         = NULL;
         dht_layout_t *new_layout   = NULL;
         dht_conf_t   *priv         = NULL;
         dht_local_t  *local        = NULL;
+        uint32_t      subvol_down  = 0;
+        int           ret          = 0;
 
         this  = frame->this;
         priv  = this->private;
         local = frame->local;
 
-        count = cnt = dht_get_layout_count (this, layout, 0);
-
-        chunk = ((unsigned long) 0xffffffff) / ((cnt) ? cnt : 1);
-
-        start_subvol = dht_selfheal_layout_alloc_start (this, loc, layout);
-
-        fix_array = GF_CALLOC (sizeof (int), layout->cnt, gf_common_mt_char);
-        if (!fix_array) {
-                /* No fix, use the existing layout itself */
+        if (layout->type == DHT_HASH_TYPE_DM_USER) {
+                gf_log (THIS->name, GF_LOG_DEBUG, "leaving %s alone",
+                        loc->path);
                 goto done;
         }
 
@@ -592,98 +751,33 @@ dht_fix_layout_of_directory (call_frame_t *frame, loc_t *loc,
         if (!new_layout)
                 goto done;
 
-        for (i = 0; i < new_layout->cnt; i++) {
-                /* TODO: fix this in layout_alloc() itself */
-                new_layout->list[i].err = -ENOENT;
-                if (i < layout->cnt)
-                        new_layout->list[i].xlator = layout->list[i].xlator;
-        }
-
-        /* Check if there are any overlap in layout, and give the proper fix */
-        for (i = 0; i < layout->cnt; i++) {
-                /* No need to fix if 'err' is not '-1' */
-                if (layout->list[i].err != -1)
-                        continue;
-
-                /* If already existing layout is having no range, skip it */
-                start = layout->list[i].start;
-                stop  = layout->list[i].stop;
-                if ((stop - start) == 0)
-                        continue;
-
-                max_overlap = 0;
-
-                /* 'j' is used as starting point of each chunk */
-                for (j = 1; j <= count; j++) {
-                        /* if chunk is already used, don't use it again */
-                        for (k = 0; k < i; k++)
-                                if (j == fix_array[k])
-                                        break;
-                        if (k < i)
-                                continue;
-
-                        overlap = dht_find_overlap (i, (j-1), start, stop, chunk);
-                        if (max_overlap < overlap) {
-                                max_overlap = overlap;
-                                fix_array[i] = j;
-                        }
-                }
+        /* If a subvolume is down, do not re-write the layout. */
+        ret = dht_layout_anomalies (this, loc, layout, NULL, NULL, NULL,
+                                    &subvol_down, NULL, NULL);
 
-                /* If we have any overlap, then use that itself as new
-                   layout for the subvolume */
-                if (fix_array[i]) {
-                        chunk_begin = chunk * (fix_array[i] - 1);
-                        new_layout->list[i].err = -1;
-                        DHT_SET_LAYOUT_RANGE (new_layout, i, chunk_begin,
-                                              chunk, cnt, loc->path);
-                        /* make sure to give (max - 1) as 'stop' range,
-                           if it is last chunk */
-                        if (fix_array[i] == count)
-                                new_layout->list[i].stop = 0xffffffff;
-                        if (--cnt == 0)
-                                goto done;
-
-                }
+        if (subvol_down || (ret == -1)) {
+                gf_log (this->name, GF_LOG_WARNING, "%u subvolume(s) are down"
+                        ". Skipping fix layout.", subvol_down);
+                GF_FREE (new_layout);
+                return NULL;
         }
 
-        /* Now, look for layouts which are not having any overlaps
-           and give it a fix */
-        for (loop_cnt = 0, i = start_subvol; loop_cnt < new_layout->cnt;
-             i++, loop_cnt++) {
-                if (i == new_layout->cnt)
-                        i = 0;
-
-                /* If 'fix_array[i]' is set, the layout is already fixed. */
-                if (fix_array[i])
-                        continue;
+        for (i = 0; i < new_layout->cnt; i++) {
+		if (layout->list[i].err != ENOSPC)
+			new_layout->list[i].err = layout->list[i].err;
+		else
+			new_layout->list[i].err = -1;
 
-                if (layout->list[i].err != -1) {
-                        new_layout->list[i].err = layout->list[i].err;
-                        continue;
-                }
+		new_layout->list[i].xlator = layout->list[i].xlator;
+        }
 
-                for (k = 1; k <= count; k++) {
-                        for (j = 0; j < new_layout->cnt; j++) {
-                                if (k == fix_array[j])
-                                        break;
-                        }
-                        /* Didn't find any of the list begining with 'k' */
-                        if (j == new_layout->cnt)
-                                break;
-                }
+	/* First give it a layout as though it is a new directory. This
+	   ensures rotation to kick in */
+        dht_layout_sort_volname (new_layout);
+	dht_selfheal_layout_new_directory (frame, loc, new_layout);
 
-                fix_array[i] = k;
-                chunk_begin = (k - 1) * chunk;
-                new_layout->list[i].err = -1;
-                DHT_SET_LAYOUT_RANGE (new_layout, i, chunk_begin, chunk, cnt,
-                                      loc->path);
-                /* make sure to give (max - 1) as 'stop' range,
-                   if it is last chunk */
-                if (k == count)
-                        new_layout->list[i].stop = 0xffffffff;
-                if (--cnt == 0)
-                        goto done;
-        }
+	/* Now selectively re-assign ranges only when it helps */
+	dht_selfheal_layout_maximize_overlap (frame, loc, new_layout, layout);
 
 done:
         if (new_layout) {
@@ -697,10 +791,7 @@ done:
                 local->layout = new_layout;
         }
 
-        if (fix_array)
-                GF_FREE (fix_array);
-
-        return new_layout;
+        return local->layout;
 }
 
 
@@ -724,9 +815,11 @@ dht_selfheal_layout_new_directory (call_frame_t *frame, loc_t *loc,
 
         start_subvol = dht_selfheal_layout_alloc_start (this, loc, layout);
 
+        /* clear out the range, as we are re-computing here */
+        DHT_RESET_LAYOUT_RANGE (layout);
         for (i = start_subvol; i < layout->cnt; i++) {
                 err = layout->list[i].err;
-                if (err == -1) {
+                if (err == -1 || err == ENOENT) {
                         DHT_SET_LAYOUT_RANGE(layout, i, start, chunk,
                                              cnt, loc->path);
                         if (--cnt == 0) {
@@ -739,7 +832,7 @@ dht_selfheal_layout_new_directory (call_frame_t *frame, loc_t *loc,
 
         for (i = 0; i < start_subvol; i++) {
                 err = layout->list[i].err;
-                if (err == -1) {
+                if (err == -1 || err == ENOENT) {
                         DHT_SET_LAYOUT_RANGE(layout, i, start, chunk,
                                              cnt, loc->path);
                         if (--cnt == 0) {
@@ -758,35 +851,17 @@ int
 dht_selfheal_dir_getafix (call_frame_t *frame, loc_t *loc,
                           dht_layout_t *layout)
 {
-        dht_conf_t  *conf = NULL;
-        xlator_t    *this = NULL;
         dht_local_t *local = NULL;
-        int          missing = -1;
-        int          down = -1;
-        int          holes = -1;
+        uint32_t     holes = 0;
         int          ret = -1;
         int          i = -1;
-        int          overlaps = -1;
+        uint32_t     overlaps = 0;
 
-        this = frame->this;
-        conf = this->private;
         local = frame->local;
 
-        missing = local->selfheal.missing;
-        down = local->selfheal.down;
         holes = local->selfheal.hole_cnt;
         overlaps = local->selfheal.overlaps_cnt;
 
-        if ((missing + down) == conf->subvolume_cnt) {
-                dht_selfheal_layout_new_directory (frame, loc, layout);
-                ret = 0;
-        }
-
-        if (holes <= down) {
-                /* the down subvol might fill up the holes */
-                ret = 0;
-        }
-
         if (holes || overlaps) {
                 dht_selfheal_layout_new_directory (frame, loc, layout);
                 ret = 0;
@@ -838,6 +913,9 @@ dht_fix_directory_layout (call_frame_t *frame,
 
         /* No layout sorting required here */
         tmp_layout = dht_fix_layout_of_directory (frame, &local->loc, layout);
+        if (!tmp_layout) {
+                return -1;
+        }
         dht_fix_dir_xattr (frame, &local->loc, tmp_layout);
 
         return 0;
@@ -860,9 +938,8 @@ dht_selfheal_directory (call_frame_t *frame, dht_selfheal_dir_cbk_t dir_cbk,
         dht_layout_anomalies (this, loc, layout,
                               &local->selfheal.hole_cnt,
                               &local->selfheal.overlaps_cnt,
-                              &local->selfheal.missing,
-                              &local->selfheal.down,
-                              &local->selfheal.misc);
+                              NULL, &local->selfheal.down,
+                              &local->selfheal.misc, NULL);
 
         down     = local->selfheal.down;
         misc     = local->selfheal.misc;
@@ -921,3 +998,51 @@ dht_selfheal_restore (call_frame_t *frame, dht_selfheal_dir_cbk_t dir_cbk,
 
         return ret;
 }
+
+int
+dht_dir_attr_heal (void *data)
+{
+        call_frame_t    *frame = NULL;
+        dht_local_t     *local = NULL;
+        xlator_t        *subvol = NULL;
+        xlator_t        *this  = NULL;
+        dht_conf_t      *conf  = NULL;
+        int              call_cnt = 0;
+        int              ret   = -1;
+        int              i     = 0;
+
+        GF_VALIDATE_OR_GOTO ("dht", data, out);
+
+        frame = data;
+        local = frame->local;
+        this = frame->this;
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO ("dht", local, out);
+        conf = this->private;
+        GF_VALIDATE_OR_GOTO ("dht", conf, out);
+
+        call_cnt = conf->subvolume_cnt;
+
+        for (i = 0; i < call_cnt; i++) {
+                subvol = conf->subvolumes[i];
+                if (!subvol || (subvol == dht_first_up_subvol (this)))
+                        continue;
+                ret = syncop_setattr (subvol, &local->loc, &local->stbuf,
+                                      (GF_SET_ATTR_UID | GF_SET_ATTR_GID),
+                                      NULL, NULL);
+                if (ret) {
+                        gf_log ("dht", GF_LOG_ERROR, "Failed to set uid/gid on"
+                                " %s on %s subvol (%s)", local->loc.path,
+                                subvol->name, strerror (-ret));
+                }
+        }
+out:
+        return 0;
+}
+
+int
+dht_dir_attr_heal_done (int ret, call_frame_t *sync_frame, void *data)
+{
+        DHT_STACK_DESTROY (sync_frame);
+        return 0;
+}
diff --git a/xlators/cluster/dht/src/dht-shared.c b/xlators/cluster/dht/src/dht-shared.c
new file mode 100644
index 000000000..36c073973
--- /dev/null
+++ b/xlators/cluster/dht/src/dht-shared.c
@@ -0,0 +1,781 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+/* TODO: add NS locking */
+
+#include "statedump.h"
+#include "dht-common.h"
+
+/* TODO:
+   - use volumename in xattr instead of "dht"
+   - use NS locks
+   - handle all cases in self heal layout reconstruction
+   - complete linkfile selfheal
+*/
+struct volume_options options[];
+
+void
+dht_layout_dump (dht_layout_t  *layout, const char *prefix)
+{
+
+        char    key[GF_DUMP_MAX_BUF_LEN];
+        int     i = 0;
+
+        if (!layout)
+                goto out;
+        if (!prefix)
+                goto out;
+
+        gf_proc_dump_build_key(key, prefix, "cnt");
+        gf_proc_dump_write(key, "%d", layout->cnt);
+        gf_proc_dump_build_key(key, prefix, "preset");
+        gf_proc_dump_write(key, "%d", layout->preset);
+        gf_proc_dump_build_key(key, prefix, "gen");
+        gf_proc_dump_write(key, "%d", layout->gen);
+        if (layout->type != IA_INVAL) {
+                gf_proc_dump_build_key(key, prefix, "inode type");
+                gf_proc_dump_write(key, "%d", layout->type);
+        }
+
+        if  (!IA_ISDIR (layout->type))
+                goto out;
+
+        for (i = 0; i < layout->cnt; i++) {
+                gf_proc_dump_build_key(key, prefix,"list[%d].err", i);
+                gf_proc_dump_write(key, "%d", layout->list[i].err);
+                gf_proc_dump_build_key(key, prefix,"list[%d].start", i);
+                gf_proc_dump_write(key, "%u", layout->list[i].start);
+                gf_proc_dump_build_key(key, prefix,"list[%d].stop", i);
+                gf_proc_dump_write(key, "%u", layout->list[i].stop);
+                if (layout->list[i].xlator) {
+                        gf_proc_dump_build_key(key, prefix,
+                                               "list[%d].xlator.type", i);
+                        gf_proc_dump_write(key, "%s",
+                                           layout->list[i].xlator->type);
+                        gf_proc_dump_build_key(key, prefix,
+                                               "list[%d].xlator.name", i);
+                        gf_proc_dump_write(key, "%s",
+                                           layout->list[i].xlator->name);
+                }
+        }
+
+out:
+        return;
+}
+
+
+int32_t
+dht_priv_dump (xlator_t *this)
+{
+        char            key_prefix[GF_DUMP_MAX_BUF_LEN];
+        char            key[GF_DUMP_MAX_BUF_LEN];
+        int             i = 0;
+        dht_conf_t      *conf = NULL;
+        int             ret = -1;
+
+        if (!this)
+                goto out;
+
+        conf = this->private;
+        if (!conf)
+                goto out;
+
+        ret = TRY_LOCK(&conf->subvolume_lock);
+        if (ret != 0) {
+                return ret;
+        }
+
+        gf_proc_dump_add_section("xlator.cluster.dht.%s.priv", this->name);
+        gf_proc_dump_build_key(key_prefix,"xlator.cluster.dht","%s.priv",
+                               this->name);
+        gf_proc_dump_write("subvol_cnt","%d", conf->subvolume_cnt);
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                snprintf (key, sizeof (key), "subvolumes[%d]", i);
+                gf_proc_dump_write(key, "%s.%s", conf->subvolumes[i]->type,
+                                   conf->subvolumes[i]->name);
+                if (conf->file_layouts && conf->file_layouts[i]){
+                        snprintf (key, sizeof (key), "file_layouts[%d]", i);
+                        dht_layout_dump(conf->file_layouts[i], key);
+                }
+                if (conf->dir_layouts && conf->dir_layouts[i]) {
+                        snprintf (key, sizeof (key), "dir_layouts[%d]", i);
+                        dht_layout_dump(conf->dir_layouts[i], key);
+                }
+                if (conf->subvolume_status) {
+
+                        snprintf (key, sizeof (key), "subvolume_status[%d]", i);
+                        gf_proc_dump_write(key, "%d",
+                                           (int)conf->subvolume_status[i]);
+                }
+
+        }
+
+        gf_proc_dump_write("search_unhashed", "%d", conf->search_unhashed);
+        gf_proc_dump_write("gen", "%d", conf->gen);
+        gf_proc_dump_write("min_free_disk", "%lf", conf->min_free_disk);
+	gf_proc_dump_write("min_free_inodes", "%lf", conf->min_free_inodes);
+        gf_proc_dump_write("disk_unit", "%c", conf->disk_unit);
+        gf_proc_dump_write("refresh_interval", "%d", conf->refresh_interval);
+        gf_proc_dump_write("unhashed_sticky_bit", "%d", conf->unhashed_sticky_bit);
+
+        if (conf->du_stats) {
+                for (i = 0; i < conf->subvolume_cnt; i++) {
+                        if (!conf->subvolume_status[i])
+                                continue;
+
+                        snprintf (key, sizeof (key), "subvolumes[%d]", i);
+                        gf_proc_dump_write (key, "%s",
+                                            conf->subvolumes[i]->name);
+
+                        snprintf (key, sizeof (key),
+                                  "du_stats[%d].avail_percent", i);
+                        gf_proc_dump_write (key, "%lf",
+                                            conf->du_stats[i].avail_percent);
+
+                        snprintf (key, sizeof (key), "du_stats[%d].avail_space",
+                                  i);
+                        gf_proc_dump_write (key, "%lu",
+                                            conf->du_stats[i].avail_space);
+
+                        snprintf (key, sizeof (key),
+                                  "du_stats[%d].avail_inodes", i);
+                        gf_proc_dump_write (key, "%lf",
+                                            conf->du_stats[i].avail_inodes);
+
+                        snprintf (key, sizeof (key), "du_stats[%d].log", i);
+                        gf_proc_dump_write (key, "%lu",
+                                            conf->du_stats[i].log);
+                }
+        }
+
+        if (conf->last_stat_fetch.tv_sec)
+                gf_proc_dump_write("last_stat_fetch", "%s",
+                                    ctime(&conf->last_stat_fetch.tv_sec));
+
+        UNLOCK(&conf->subvolume_lock);
+
+out:
+        return ret;
+}
+
+int32_t
+dht_inodectx_dump (xlator_t *this, inode_t *inode)
+{
+        int             ret = -1;
+        dht_layout_t    *layout = NULL;
+
+        if (!this)
+                goto out;
+        if (!inode)
+                goto out;
+
+        ret = dht_inode_ctx_layout_get (inode, this, &layout);
+
+        if ((ret != 0) || !layout)
+                return ret;
+
+        gf_proc_dump_add_section("xlator.cluster.dht.%s.inode", this->name);
+        dht_layout_dump(layout, "layout");
+
+out:
+        return ret;
+}
+
+void
+dht_fini (xlator_t *this)
+{
+        int         i = 0;
+        dht_conf_t *conf = NULL;
+
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+
+        conf = this->private;
+        this->private = NULL;
+        if (conf) {
+                if (conf->file_layouts) {
+                        for (i = 0; i < conf->subvolume_cnt; i++) {
+                                GF_FREE (conf->file_layouts[i]);
+                        }
+                        GF_FREE (conf->file_layouts);
+                }
+
+                GF_FREE (conf->subvolumes);
+
+                GF_FREE (conf->subvolume_status);
+
+                GF_FREE (conf);
+        }
+out:
+        return;
+}
+
+int32_t
+mem_acct_init (xlator_t *this)
+{
+        int     ret = -1;
+
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+
+        ret = xlator_mem_acct_init (this, gf_dht_mt_end + 1);
+
+        if (ret != 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
+                        "failed");
+                return ret;
+        }
+out:
+        return ret;
+}
+
+
+int
+dht_parse_decommissioned_bricks (xlator_t *this, dht_conf_t *conf,
+                                 const char *bricks)
+{
+        int         i  = 0;
+        int         ret  = -1;
+        char       *tmpstr = NULL;
+        char       *dup_brick = NULL;
+        char       *node = NULL;
+
+        if (!conf || !bricks)
+                goto out;
+
+        dup_brick = gf_strdup (bricks);
+        node = strtok_r (dup_brick, ",", &tmpstr);
+        while (node) {
+                for (i = 0; i < conf->subvolume_cnt; i++) {
+                        if (!strcmp (conf->subvolumes[i]->name, node)) {
+                                conf->decommissioned_bricks[i] =
+                                        conf->subvolumes[i];
+                                        conf->decommission_subvols_cnt++;
+                                gf_log (this->name, GF_LOG_INFO,
+                                        "decommissioning subvolume %s",
+                                        conf->subvolumes[i]->name);
+                                break;
+                        }
+                }
+                if (i == conf->subvolume_cnt) {
+                        /* Wrong node given. */
+                        goto out;
+                }
+                node = strtok_r (NULL, ",", &tmpstr);
+        }
+
+        ret = 0;
+        conf->decommission_in_progress = 1;
+out:
+        GF_FREE (dup_brick);
+
+        return ret;
+}
+
+
+int
+dht_decommissioned_remove (xlator_t *this, dht_conf_t *conf)
+{
+        int         i  = 0;
+        int         ret  = -1;
+
+        if (!conf)
+                goto out;
+
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (conf->decommissioned_bricks[i]) {
+                        conf->decommissioned_bricks[i] = NULL;
+                        conf->decommission_subvols_cnt--;
+                }
+        }
+
+        ret = 0;
+out:
+
+        return ret;
+}
+void
+dht_init_regex (xlator_t *this, dict_t *odict, char *name,
+                regex_t *re, gf_boolean_t *re_valid)
+{
+        char    *temp_str;
+
+        if (dict_get_str (odict, name, &temp_str) != 0) {
+                if (strcmp(name,"rsync-hash-regex")) {
+                        return;
+                }
+                temp_str = "^\\.(.+)\\.[^.]+$";
+        }
+
+        if (*re_valid) {
+                regfree(re);
+                *re_valid = _gf_false;
+        }
+
+        if (!strcmp(temp_str,"none")) {
+                return;
+        }
+
+        if (regcomp(re,temp_str,REG_EXTENDED) == 0) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "using regex %s = %s", name, temp_str);
+                *re_valid = _gf_true;
+        }
+        else {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "compiling regex %s failed", temp_str);
+        }
+}
+
+int
+dht_reconfigure (xlator_t *this, dict_t *options)
+{
+        dht_conf_t      *conf = NULL;
+        char            *temp_str = NULL;
+        gf_boolean_t     search_unhashed;
+        int              ret = -1;
+
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO ("dht", options, out);
+
+        conf = this->private;
+        if (!conf)
+                return 0;
+
+        if (dict_get_str (options, "lookup-unhashed", &temp_str) == 0) {
+                /* If option is not "auto", other options _should_ be boolean*/
+                if (strcasecmp (temp_str, "auto")) {
+                        if (!gf_string2boolean (temp_str, &search_unhashed)) {
+                                gf_log(this->name, GF_LOG_DEBUG, "Reconfigure:"
+                                       " lookup-unhashed reconfigured (%s)",
+                                       temp_str);
+                                conf->search_unhashed = search_unhashed;
+                        } else {
+                                gf_log(this->name, GF_LOG_ERROR, "Reconfigure:"
+                                       " lookup-unhashed should be boolean,"
+                                       " not (%s), defaulting to (%d)",
+                                       temp_str, conf->search_unhashed);
+                                //return -1;
+                                ret = -1;
+                                goto out;
+                        }
+                } else {
+                        gf_log(this->name, GF_LOG_DEBUG, "Reconfigure:"
+                               " lookup-unhashed reconfigured auto ");
+                        conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_AUTO;
+                }
+        }
+
+	GF_OPTION_RECONF ("min-free-disk", conf->min_free_disk, options,
+                          percent_or_size, out);
+        /* option can be any one of percent or bytes */
+        conf->disk_unit = 0;
+        if (conf->min_free_disk < 100.0)
+                conf->disk_unit = 'p';
+
+	GF_OPTION_RECONF ("min-free-inodes", conf->min_free_inodes, options,
+                          percent, out);
+
+        GF_OPTION_RECONF ("directory-layout-spread", conf->dir_spread_cnt,
+                          options, uint32, out);
+
+        GF_OPTION_RECONF ("readdir-optimize", conf->readdir_optimize, options,
+                          bool, out);
+        if (conf->defrag) {
+                GF_OPTION_RECONF ("rebalance-stats", conf->defrag->stats,
+                                  options, bool, out);
+        }
+
+        if (dict_get_str (options, "decommissioned-bricks", &temp_str) == 0) {
+                ret = dht_parse_decommissioned_bricks (this, conf, temp_str);
+                if (ret == -1)
+                        goto out;
+        } else {
+                ret = dht_decommissioned_remove (this, conf);
+                if (ret == -1)
+                        goto out;
+        }
+
+        dht_init_regex (this, options, "rsync-hash-regex",
+                        &conf->rsync_regex, &conf->rsync_regex_valid);
+        dht_init_regex (this, options, "extra-hash-regex",
+                        &conf->extra_regex, &conf->extra_regex_valid);
+
+        ret = 0;
+out:
+        return ret;
+}
+
+static int
+gf_defrag_pattern_list_fill (xlator_t *this, gf_defrag_info_t *defrag, char *data)
+{
+        int                    ret = -1;
+        char                  *tmp_str = NULL;
+        char                  *tmp_str1 = NULL;
+        char                  *dup_str = NULL;
+        char                  *num = NULL;
+        char                  *pattern_str = NULL;
+        char                  *pattern = NULL;
+        gf_defrag_pattern_list_t *temp_list = NULL;
+        gf_defrag_pattern_list_t *pattern_list = NULL;
+
+        if (!this || !defrag || !data)
+                goto out;
+
+        /* Get the pattern for pattern list. "pattern:<optional-size>"
+         * eg: *avi, *pdf:10MB, *:1TB
+         */
+        pattern_str = strtok_r (data, ",", &tmp_str);
+        while (pattern_str) {
+                dup_str = gf_strdup (pattern_str);
+                pattern_list = GF_CALLOC (1, sizeof (gf_defrag_pattern_list_t),
+                                        1);
+                if (!pattern_list) {
+                        goto out;
+                }
+                pattern = strtok_r (dup_str, ":", &tmp_str1);
+                num = strtok_r (NULL, ":", &tmp_str1);
+                if (!pattern)
+                        goto out;
+                if (!num) {
+                        if (gf_string2bytesize(pattern, &pattern_list->size)
+                             == 0) {
+                                pattern = "*";
+                        }
+                } else if (gf_string2bytesize (num, &pattern_list->size) != 0) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "invalid number format \"%s\"", num);
+                        goto out;
+                }
+                memcpy (pattern_list->path_pattern, pattern, strlen (dup_str));
+
+                if (!defrag->defrag_pattern)
+                        temp_list = NULL;
+                else
+                        temp_list = defrag->defrag_pattern;
+
+                pattern_list->next = temp_list;
+
+                defrag->defrag_pattern = pattern_list;
+                pattern_list = NULL;
+
+                GF_FREE (dup_str);
+                dup_str = NULL;
+
+                pattern_str = strtok_r (NULL, ",", &tmp_str);
+        }
+
+        ret = 0;
+out:
+        if (ret)
+                GF_FREE (pattern_list);
+        GF_FREE (dup_str);
+
+        return ret;
+}
+
+int
+dht_init (xlator_t *this)
+{
+        dht_conf_t                      *conf           = NULL;
+        char                            *temp_str       = NULL;
+        int                              ret            = -1;
+        int                              i              = 0;
+        gf_defrag_info_t                *defrag         = NULL;
+        int                              cmd            = 0;
+        char                            *node_uuid      = NULL;
+
+
+        GF_VALIDATE_OR_GOTO ("dht", this, err);
+
+        if (!this->children) {
+                gf_log (this->name, GF_LOG_CRITICAL,
+                        "Distribute needs more than one subvolume");
+                return -1;
+        }
+
+        if (!this->parents) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "dangling volume. check volfile");
+        }
+
+        conf = GF_CALLOC (1, sizeof (*conf), gf_dht_mt_dht_conf_t);
+        if (!conf) {
+                goto err;
+        }
+
+        ret = dict_get_int32 (this->options, "rebalance-cmd", &cmd);
+
+        if (cmd) {
+                defrag = GF_CALLOC (1, sizeof (gf_defrag_info_t),
+                                    gf_defrag_info_mt);
+
+                GF_VALIDATE_OR_GOTO (this->name, defrag, err);
+
+                LOCK_INIT (&defrag->lock);
+
+                defrag->is_exiting = 0;
+
+                conf->defrag = defrag;
+
+                ret = dict_get_str (this->options, "node-uuid", &node_uuid);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "node-uuid not "
+                                "specified");
+                        goto err;
+                }
+
+                if (uuid_parse (node_uuid, defrag->node_uuid)) {
+                        gf_log (this->name, GF_LOG_ERROR, "Cannot parse "
+                                "glusterd node uuid");
+                        goto err;
+                }
+
+                defrag->cmd = cmd;
+
+                defrag->stats = _gf_false;
+        }
+
+        conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_ON;
+        if (dict_get_str (this->options, "lookup-unhashed", &temp_str) == 0) {
+                /* If option is not "auto", other options _should_ be boolean */
+                if (strcasecmp (temp_str, "auto"))
+                        gf_string2boolean (temp_str, &conf->search_unhashed);
+                else
+                        conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_AUTO;
+        }
+
+        GF_OPTION_INIT ("unhashed-sticky-bit", conf->unhashed_sticky_bit, bool,
+                        err);
+
+        GF_OPTION_INIT ("use-readdirp", conf->use_readdirp, bool, err);
+
+	GF_OPTION_INIT ("min-free-disk", conf->min_free_disk, percent_or_size,
+			err);
+
+        GF_OPTION_INIT ("min-free-inodes", conf->min_free_inodes, percent,
+                        err);
+
+        conf->dir_spread_cnt = conf->subvolume_cnt;
+        GF_OPTION_INIT ("directory-layout-spread", conf->dir_spread_cnt,
+                        uint32, err);
+
+        GF_OPTION_INIT ("assert-no-child-down", conf->assert_no_child_down,
+                        bool, err);
+
+        GF_OPTION_INIT ("readdir-optimize", conf->readdir_optimize, bool, err);
+
+        if (defrag) {
+                GF_OPTION_INIT ("rebalance-stats", defrag->stats, bool, err);
+                if (dict_get_str (this->options, "rebalance-filter", &temp_str)
+                    == 0) {
+                        if (gf_defrag_pattern_list_fill (this, defrag, temp_str)
+                            == -1) {
+                                gf_log (this->name, GF_LOG_ERROR, "Cannot parse"
+                                        " rebalance-filter (%s)", temp_str);
+                                goto err;
+                        }
+                }
+        }
+
+        /* option can be any one of percent or bytes */
+        conf->disk_unit = 0;
+        if (conf->min_free_disk < 100)
+                conf->disk_unit = 'p';
+
+        ret = dht_init_subvolumes (this, conf);
+        if (ret == -1) {
+                goto err;
+        }
+
+        if (dict_get_str (this->options, "decommissioned-bricks", &temp_str) == 0) {
+                ret = dht_parse_decommissioned_bricks (this, conf, temp_str);
+                if (ret == -1)
+                        goto err;
+        }
+
+        dht_init_regex (this, this->options, "rsync-hash-regex",
+                        &conf->rsync_regex, &conf->rsync_regex_valid);
+        dht_init_regex (this, this->options, "extra-hash-regex",
+                        &conf->extra_regex, &conf->extra_regex_valid);
+
+        ret = dht_layouts_init (this, conf);
+        if (ret == -1) {
+                goto err;
+        }
+
+        LOCK_INIT (&conf->subvolume_lock);
+        LOCK_INIT (&conf->layout_lock);
+
+        conf->gen = 1;
+
+        this->local_pool = mem_pool_new (dht_local_t, 512);
+        if (!this->local_pool) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to create local_t's memory pool");
+                goto err;
+        }
+
+        GF_OPTION_INIT ("xattr-name", conf->xattr_name, str, err);
+        gf_asprintf (&conf->link_xattr_name, "%s."DHT_LINKFILE_STR,
+                     conf->xattr_name);
+        gf_asprintf (&conf->wild_xattr_name, "%s*", conf->xattr_name);
+        if (!conf->link_xattr_name || !conf->wild_xattr_name) {
+                goto err;
+        }
+
+        this->private = conf;
+
+        return 0;
+
+err:
+        if (conf) {
+                if (conf->file_layouts) {
+                        for (i = 0; i < conf->subvolume_cnt; i++) {
+                                GF_FREE (conf->file_layouts[i]);
+                        }
+                        GF_FREE (conf->file_layouts);
+                }
+
+                GF_FREE (conf->subvolumes);
+
+                GF_FREE (conf->subvolume_status);
+
+                GF_FREE (conf->du_stats);
+
+                GF_FREE (conf->defrag);
+
+                GF_FREE (conf->xattr_name);
+                GF_FREE (conf->link_xattr_name);
+                GF_FREE (conf->wild_xattr_name);
+
+                GF_FREE (conf);
+        }
+
+        return -1;
+}
+
+
+struct volume_options options[] = {
+        { .key  = {"lookup-unhashed"},
+          .value = {"auto", "yes", "no", "enable", "disable", "1", "0",
+                    "on", "off"},
+          .type = GF_OPTION_TYPE_STR,
+          .default_value = "on",
+          .description = "This option if set to ON, does a lookup through "
+          "all the sub-volumes, in case a lookup didn't return any result "
+          "from the hash subvolume. If set to OFF, it does not do a lookup "
+          "on the remaining subvolumes."
+        },
+        { .key  = {"min-free-disk"},
+          .type = GF_OPTION_TYPE_PERCENT_OR_SIZET,
+          .default_value = "10%",
+          .description = "Percentage/Size of disk space, after which the "
+          "process starts balancing out the cluster, and logs will appear "
+          "in log files",
+        },
+	{ .key  = {"min-free-inodes"},
+          .type = GF_OPTION_TYPE_PERCENT,
+          .default_value = "5%",
+          .description = "after system has only N% of inodes, warnings "
+          "starts to appear in log files",
+        },
+        { .key = {"unhashed-sticky-bit"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
+        },
+        { .key = {"use-readdirp"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "on",
+          .description = "This option if set to ON, forces the use of "
+          "readdirp, and hence also displays the stats of the files."
+        },
+        { .key = {"assert-no-child-down"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
+          .description = "This option if set to ON, in the event of "
+          "CHILD_DOWN, will call exit."
+        },
+        { .key  = {"directory-layout-spread"},
+          .type = GF_OPTION_TYPE_INT,
+          .min  = 1,
+          .validate = GF_OPT_VALIDATE_MIN,
+          .description = "Specifies the directory layout spread. Takes number "
+                         "of subvolumes as default value."
+        },
+        { .key  = {"decommissioned-bricks"},
+          .type = GF_OPTION_TYPE_ANY,
+          .description = "This option if set to ON, decommissions "
+          "the brick, so that no new data is allowed to be created "
+          "on that brick."
+        },
+        { .key  = {"rebalance-cmd"},
+          .type = GF_OPTION_TYPE_INT,
+        },
+        { .key = {"node-uuid"},
+          .type = GF_OPTION_TYPE_STR,
+        },
+        { .key = {"rebalance-stats"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
+          .description = "This option if set to ON displays and logs the "
+          " time taken for migration of each file, during the rebalance "
+          "process. If set to OFF, the rebalance logs will only display the "
+          "time spent in each directory."
+        },
+        { .key = {"readdir-optimize"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
+          .description = "This option if set to ON enables the optimization "
+          "that allows DHT to requests non-first subvolumes to filter out "
+          "directory entries."
+        },
+        { .key = {"rsync-hash-regex"},
+          .type = GF_OPTION_TYPE_STR,
+          /* Setting a default here doesn't work.  See dht_init_regex. */
+          .description = "Regular expression for stripping temporary-file "
+          "suffix and prefix used by rsync, to prevent relocation when the "
+          "file is renamed."
+        },
+        { .key = {"extra-hash-regex"},
+          .type = GF_OPTION_TYPE_STR,
+          /* Setting a default here doesn't work.  See dht_init_regex. */
+          .description = "Regular expression for stripping temporary-file "
+          "suffix and prefix used by an application, to prevent relocation when "
+          "the file is renamed."
+        },
+        { .key = {"rebalance-filter"},
+          .type = GF_OPTION_TYPE_STR,
+        },
+
+        { .key = {"xattr-name"},
+          .type = GF_OPTION_TYPE_STR,
+          .default_value = "trusted.glusterfs.dht",
+          .description = "Base for extended attributes used by this "
+          "translator instance, to avoid conflicts with others above or "
+          "below it."
+        },
+
+        /* NUFA option */
+        { .key  = {"local-volume-name"},
+          .type = GF_OPTION_TYPE_XLATOR
+        },
+
+        /* switch option */
+        { .key  = {"pattern.switch.case"},
+          .type = GF_OPTION_TYPE_ANY
+        },
+
+        { .key  = {NULL} },
+};
diff --git a/xlators/cluster/dht/src/dht.c b/xlators/cluster/dht/src/dht.c
index 68a8efcde..fc0ca2f77 100644
--- a/xlators/cluster/dht/src/dht.c
+++ b/xlators/cluster/dht/src/dht.c
@@ -14,482 +14,15 @@
 #include "config.h"
 #endif
 
-/* TODO: add NS locking */
-
 #include "statedump.h"
 #include "dht-common.h"
 
-/* TODO:
-   - use volumename in xattr instead of "dht"
-   - use NS locks
-   - handle all cases in self heal layout reconstruction
-   - complete linkfile selfheal
-*/
-struct volume_options options[];
-
-void
-dht_layout_dump (dht_layout_t  *layout, const char *prefix)
-{
-
-        char    key[GF_DUMP_MAX_BUF_LEN];
-        int     i = 0;
-
-        GF_VALIDATE_OR_GOTO ("dht", layout, out);
-        GF_VALIDATE_OR_GOTO ("dht", prefix, out);
-
-        gf_proc_dump_build_key(key, prefix, "cnt");
-        gf_proc_dump_write(key, "%d", layout->cnt);
-        gf_proc_dump_build_key(key, prefix, "preset");
-        gf_proc_dump_write(key, "%d", layout->preset);
-        gf_proc_dump_build_key(key, prefix, "gen");
-        gf_proc_dump_write(key, "%d", layout->gen);
-        gf_proc_dump_build_key(key, prefix, "type");
-        gf_proc_dump_write(key, "%d", layout->type);
-
-        for (i = 0; i < layout->cnt; i++) {
-                gf_proc_dump_build_key(key, prefix,"list[%d].err", i);
-                gf_proc_dump_write(key, "%d", layout->list[i].err);
-                gf_proc_dump_build_key(key, prefix,"list[%d].start", i);
-                gf_proc_dump_write(key, "%u", layout->list[i].start);
-                gf_proc_dump_build_key(key, prefix,"list[%d].stop", i);
-                gf_proc_dump_write(key, "%u", layout->list[i].stop);
-                if (layout->list[i].xlator) {
-                        gf_proc_dump_build_key(key, prefix,
-                                               "list[%d].xlator.type", i);
-                        gf_proc_dump_write(key, "%s",
-                                           layout->list[i].xlator->type);
-                        gf_proc_dump_build_key(key, prefix,
-                                               "list[%d].xlator.name", i);
-                        gf_proc_dump_write(key, "%s",
-                                           layout->list[i].xlator->name);
-                }
-        }
-
-out:
-        return;
-}
-
-
-int32_t
-dht_priv_dump (xlator_t *this)
-{
-        char            key_prefix[GF_DUMP_MAX_BUF_LEN];
-        char            key[GF_DUMP_MAX_BUF_LEN];
-        int             i = 0;
-        dht_conf_t      *conf = NULL;
-        int             ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-
-        conf = this->private;
-
-        if (!conf)
-                return -1;
-
-        ret = TRY_LOCK(&conf->subvolume_lock);
-
-        if (ret != 0) {
-                gf_log("", GF_LOG_WARNING, "Unable to lock dht subvolume %s",
-                       this->name);
-                return ret;
-        }
-
-        gf_proc_dump_add_section("xlator.cluster.dht.%s.priv", this->name);
-        gf_proc_dump_build_key(key_prefix,"xlator.cluster.dht","%s.priv",
-                               this->name);
-        gf_proc_dump_write("subvol_cnt","%d", conf->subvolume_cnt);
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                sprintf (key, "subvolumes[%d]", i);
-                gf_proc_dump_write(key, "%s.%s", conf->subvolumes[i]->type,
-                                   conf->subvolumes[i]->name);
-                if (conf->file_layouts && conf->file_layouts[i]){
-                        sprintf (key, "file_layouts[%d]", i);
-                        dht_layout_dump(conf->file_layouts[i], key);
-                }
-                if (conf->dir_layouts && conf->dir_layouts[i]) {
-                        sprintf (key, "dir_layouts[%d]", i);
-                        dht_layout_dump(conf->dir_layouts[i], key);
-                }
-                if (conf->subvolume_status) {
-
-                        sprintf (key, "subvolume_status[%d]", i);
-                        gf_proc_dump_write(key, "%d",
-                                           (int)conf->subvolume_status[i]);
-                }
-
-        }
-
-        gf_proc_dump_write("search_unhashed", "%d", conf->search_unhashed);
-        gf_proc_dump_write("gen", "%d", conf->gen);
-        gf_proc_dump_write("min_free_disk", "%lu", conf->min_free_disk);
-	gf_proc_dump_write("min_free_inodes", "%lu", conf->min_free_inodes);
-        gf_proc_dump_write("disk_unit", "%c", conf->disk_unit);
-        gf_proc_dump_write("refresh_interval", "%d", conf->refresh_interval);
-        gf_proc_dump_write("unhashed_sticky_bit", "%d", conf->unhashed_sticky_bit);
-        if (conf ->du_stats) {
-                gf_proc_dump_write("du_stats.avail_percent", "%lf",
-                                   conf->du_stats->avail_percent);
-                gf_proc_dump_write("du_stats.avail_space", "%lu",
-                                   conf->du_stats->avail_space);
-		gf_proc_dump_write("du_stats.avail_inodes", "%lf",
-                                   conf->du_stats->avail_inodes);
-                gf_proc_dump_write("du_stats.log", "%lu", conf->du_stats->log);
-        }
-
-        if (conf->last_stat_fetch.tv_sec)
-                gf_proc_dump_write("last_stat_fetch", "%s",
-                                    ctime(&conf->last_stat_fetch.tv_sec));
-
-        UNLOCK(&conf->subvolume_lock);
-
-out:
-        return ret;
-}
-
-int32_t
-dht_inodectx_dump (xlator_t *this, inode_t *inode)
-{
-        int             ret = -1;
-        dht_layout_t    *layout = NULL;
-        uint64_t        tmp_layout = 0;
-
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", inode, out);
-
-        ret = inode_ctx_get (inode, this, &tmp_layout);
-
-        if (ret != 0)
-                return ret;
-
-        layout = (dht_layout_t *)(long)tmp_layout;
-
-        if (!layout)
-                return -1;
-
-        gf_proc_dump_add_section("xlator.cluster.dht.%s.inode", this->name);
-        dht_layout_dump(layout, "layout");
-
-out:
-        return ret;
-}
-
-int
-notify (xlator_t *this, int event, void *data, ...)
-{
-        int              ret = -1;
-        va_list          ap;
-        dict_t          *output = NULL;
-
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-
-
-        if (!data)
-                goto out;
-
-        va_start (ap, data);
-        output = va_arg (ap, dict_t*);
-
-        ret = dht_notify (this, event, data, output);
-
-out:
-        return ret;
-}
-
-void
-fini (xlator_t *this)
-{
-        int         i = 0;
-        dht_conf_t *conf = NULL;
-
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-
-        conf = this->private;
-        this->private = NULL;
-        if (conf) {
-                if (conf->file_layouts) {
-                        for (i = 0; i < conf->subvolume_cnt; i++) {
-                                GF_FREE (conf->file_layouts[i]);
-                        }
-                        GF_FREE (conf->file_layouts);
-                }
-
-                if (conf->subvolumes)
-                        GF_FREE (conf->subvolumes);
-
-                if (conf->subvolume_status)
-                        GF_FREE (conf->subvolume_status);
-
-                GF_FREE (conf);
-        }
-out:
-        return;
-}
-
-int32_t
-mem_acct_init (xlator_t *this)
-{
-        int     ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-
-        ret = xlator_mem_acct_init (this, gf_dht_mt_end + 1);
-
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
-                        "failed");
-                return ret;
-        }
-out:
-        return ret;
-}
-
-
-int
-dht_parse_decommissioned_bricks (xlator_t *this, dht_conf_t *conf,
-                                 const char *bricks)
-{
-        int         i  = 0;
-        int         ret  = -1;
-        char       *tmpstr = NULL;
-        char       *dup_brick = NULL;
-        char       *node = NULL;
-
-        if (!conf || !bricks)
-                goto out;
-
-        dup_brick = gf_strdup (bricks);
-        node = strtok_r (dup_brick, ",", &tmpstr);
-        while (node) {
-                for (i = 0; i < conf->subvolume_cnt; i++) {
-                        if (!strcmp (conf->subvolumes[i]->name, node)) {
-                                conf->decommissioned_bricks[i] =
-                                        conf->subvolumes[i];
-                                gf_log (this->name, GF_LOG_INFO,
-                                        "decommissioning subvolume %s",
-                                        conf->subvolumes[i]->name);
-                                break;
-                        }
-                }
-                if (i == conf->subvolume_cnt) {
-                        /* Wrong node given. */
-                        goto out;
-                }
-                node = strtok_r (NULL, ",", &tmpstr);
-        }
-
-        ret = 0;
-        conf->decommission_in_progress = 1;
-out:
-        if (dup_brick)
-                GF_FREE (dup_brick);
-
-        return ret;
-}
-
-int
-reconfigure (xlator_t *this, dict_t *options)
-{
-        dht_conf_t      *conf = NULL;
-        char            *temp_str = NULL;
-        gf_boolean_t     search_unhashed;
-        int              ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", options, out);
-
-        conf = this->private;
-        if (!conf)
-                return 0;
-
-        if (dict_get_str (options, "lookup-unhashed", &temp_str) == 0) {
-                /* If option is not "auto", other options _should_ be boolean*/
-                if (strcasecmp (temp_str, "auto")) {
-                        if (!gf_string2boolean (temp_str, &search_unhashed)) {
-                                gf_log(this->name, GF_LOG_DEBUG, "Reconfigure:"
-                                       " lookup-unhashed reconfigured (%s)",
-                                       temp_str);
-                                conf->search_unhashed = search_unhashed;
-                        } else {
-                                gf_log(this->name, GF_LOG_ERROR, "Reconfigure:"
-                                       " lookup-unhashed should be boolean,"
-                                       " not (%s), defaulting to (%d)",
-                                       temp_str, conf->search_unhashed);
-                                //return -1;
-                                ret = -1;
-                                goto out;
-                        }
-                } else {
-                        gf_log(this->name, GF_LOG_DEBUG, "Reconfigure:"
-                               " lookup-unhashed reconfigured auto ");
-                        conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_AUTO;
-                }
-        }
-
-	GF_OPTION_RECONF ("min-free-disk", conf->min_free_disk, options,
-                          percent_or_size, out);
-	GF_OPTION_RECONF ("min-free-inodes", conf->min_free_inodes, options,
-                          percent, out);
-        GF_OPTION_RECONF ("directory-layout-spread", conf->dir_spread_cnt,
-                          options, uint32, out);
-        GF_OPTION_RECONF ("readdir-optimize", conf->readdir_optimize, options,
-                          bool, out);
-        if (dict_get_str (options, "decommissioned-bricks", &temp_str) == 0) {
-                ret = dht_parse_decommissioned_bricks (this, conf, temp_str);
-                if (ret == -1)
-                        goto out;
-        }
-
-        ret = 0;
-out:
-        return ret;
-}
-
-
-int
-init (xlator_t *this)
-{
-        dht_conf_t                      *conf           = NULL;
-        char                            *temp_str       = NULL;
-        int                              ret            = -1;
-        int                              i              = 0;
-        gf_defrag_info_t                *defrag         = NULL;
-        int                              cmd            = 0;
-        char                            *node_uuid      = NULL;
-
-
-        GF_VALIDATE_OR_GOTO ("dht", this, err);
-
-        if (!this->children) {
-                gf_log (this->name, GF_LOG_CRITICAL,
-                        "Distribute needs more than one subvolume");
-                return -1;
-        }
-
-        if (!this->parents) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "dangling volume. check volfile");
-        }
-
-        conf = GF_CALLOC (1, sizeof (*conf), gf_dht_mt_dht_conf_t);
-        if (!conf) {
-                goto err;
-        }
-
-        ret = dict_get_int32 (this->options, "rebalance-cmd", &cmd);
-
-        if (cmd) {
-                defrag = GF_CALLOC (1, sizeof (gf_defrag_info_t),
-                                    gf_defrag_info_mt);
-
-                GF_VALIDATE_OR_GOTO (this->name, defrag, err);
-
-                LOCK_INIT (&defrag->lock);
-
-                defrag->is_exiting = 0;
-
-                ret = dict_get_str (this->options, "node-uuid", &node_uuid);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "node-uuid not "
-                                "specified");
-                        goto err;
-                }
-
-                if (uuid_parse (node_uuid, defrag->node_uuid)) {
-                        gf_log (this->name, GF_LOG_ERROR, "Cannot parse "
-                                "glusterd node uuid");
-                        goto err;
-                }
-
-                defrag->cmd = cmd;
-
-                conf->defrag = defrag;
-        }
-
-
-        conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_ON;
-        if (dict_get_str (this->options, "lookup-unhashed", &temp_str) == 0) {
-                /* If option is not "auto", other options _should_ be boolean */
-                if (strcasecmp (temp_str, "auto"))
-                        gf_string2boolean (temp_str, &conf->search_unhashed);
-                else
-                        conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_AUTO;
-        }
-
-        GF_OPTION_INIT ("unhashed-sticky-bit", conf->unhashed_sticky_bit, bool,
-                        err);
-
-        GF_OPTION_INIT ("use-readdirp", conf->use_readdirp, bool, err);
-
-	GF_OPTION_INIT ("min-free-disk", conf->min_free_disk, percent_or_size,
-			err);
-
-        GF_OPTION_INIT ("min-free-inodes", conf->min_free_inodes, percent,
-                        err);
-
-        conf->dir_spread_cnt = conf->subvolume_cnt;
-        GF_OPTION_INIT ("directory-layout-spread", conf->dir_spread_cnt,
-                        uint32, err);
-
-        GF_OPTION_INIT ("assert-no-child-down", conf->assert_no_child_down,
-                        bool, err);
-        GF_OPTION_INIT ("readdir-optimize", conf->readdir_optimize, bool, err);
-
-        ret = dht_init_subvolumes (this, conf);
-        if (ret == -1) {
-                goto err;
-        }
-
-        if (dict_get_str (this->options, "decommissioned-bricks", &temp_str) == 0) {
-                ret = dht_parse_decommissioned_bricks (this, conf, temp_str);
-                if (ret == -1)
-                        goto err;
-        }
-
-        ret = dht_layouts_init (this, conf);
-        if (ret == -1) {
-                goto err;
-        }
-
-        LOCK_INIT (&conf->subvolume_lock);
-        LOCK_INIT (&conf->layout_lock);
-
-        conf->gen = 1;
-
-        this->local_pool = mem_pool_new (dht_local_t, 512);
-        if (!this->local_pool) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to create local_t's memory pool");
-                goto err;
-        }
-
-        this->private = conf;
-
-        return 0;
-
-err:
-        if (conf) {
-                if (conf->file_layouts) {
-                        for (i = 0; i < conf->subvolume_cnt; i++) {
-                                GF_FREE (conf->file_layouts[i]);
-                        }
-                        GF_FREE (conf->file_layouts);
-                }
-
-                if (conf->subvolumes)
-                        GF_FREE (conf->subvolumes);
-
-                if (conf->subvolume_status)
-                        GF_FREE (conf->subvolume_status);
-
-                if (conf->du_stats)
-                        GF_FREE (conf->du_stats);
-
-                GF_FREE (conf);
-        }
-
-        return -1;
-}
-
+class_methods_t class_methods = {
+        .init           = dht_init,
+        .fini           = dht_fini,
+        .reconfigure    = dht_reconfigure,
+        .notify         = dht_notify
+};
 
 struct xlator_fops fops = {
         .lookup      = dht_lookup,
@@ -537,6 +70,9 @@ struct xlator_fops fops = {
         .fxattrop    = dht_fxattrop,
         .setattr     = dht_setattr,
         .fsetattr    = dht_fsetattr,
+	.fallocate   = dht_fallocate,
+	.discard     = dht_discard,
+        .zerofill    = dht_zerofill,
 };
 
 struct xlator_dumpops dumpops = {
@@ -550,55 +86,4 @@ struct xlator_cbks cbks = {
 //      .releasedir = dht_releasedir,
         .forget     = dht_forget
 };
-
-
-struct volume_options options[] = {
-        { .key  = {"lookup-unhashed"},
-          .value = {"auto", "yes", "no", "enable", "disable", "1", "0",
-                    "on", "off"},
-          .type = GF_OPTION_TYPE_STR,
-          .default_value = "on",
-        },
-        { .key  = {"min-free-disk"},
-          .type = GF_OPTION_TYPE_PERCENT_OR_SIZET,
-          .default_value = "10%",
-          .description = "Percentage/Size of disk space that must be "
-                         "kept free."
-        },
-	{ .key  = {"min-free-inodes"},
-          .type = GF_OPTION_TYPE_PERCENT,
-          .default_value = "5%",
-          .description = "Percentage inodes that must be "
-                         "kept free."
-        },
-        { .key = {"unhashed-sticky-bit"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-        },
-        { .key = {"use-readdirp"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
-        },
-        { .key = {"assert-no-child-down"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-        },
-        { .key  = {"directory-layout-spread"},
-          .type = GF_OPTION_TYPE_INT,
-        },
-        { .key  = {"decommissioned-bricks"},
-          .type = GF_OPTION_TYPE_ANY,
-        },
-        { .key  = {"rebalance-cmd"},
-          .type = GF_OPTION_TYPE_INT,
-        },
-        { .key = {"node-uuid"},
-          .type = GF_OPTION_TYPE_STR,
-        },
-        { .key = {"readdir-optimize"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-        },
-
-        { .key  = {NULL} },
-};
+;
diff --git a/xlators/cluster/dht/src/nufa.c b/xlators/cluster/dht/src/nufa.c
index 2179870d9..e934acdf0 100644
--- a/xlators/cluster/dht/src/nufa.c
+++ b/xlators/cluster/dht/src/nufa.c
@@ -18,6 +18,8 @@
 
 /* TODO: all 'TODO's in dht.c holds good */
 
+extern struct volume_options options[];
+
 int
 nufa_local_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                        int op_ret, int op_errno,
@@ -52,7 +54,8 @@ nufa_local_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if (op_ret == -1)
                 goto out;
 
-        is_linkfile = check_is_linkfile (inode, stbuf, xattr);
+        is_linkfile = check_is_linkfile (inode, stbuf, xattr,
+                                         conf->link_xattr_name);
         is_dir      = check_is_dir (inode, stbuf, xattr);
 
         if (!is_dir && !is_linkfile) {
@@ -201,7 +204,7 @@ nufa_lookup (call_frame_t *frame, xlator_t *this,
                  *       revalidates directly go to the cached-subvolume.
                  */
                 ret = dict_set_uint32 (local->xattr_req,
-                                       "trusted.glusterfs.dht", 4 * 4);
+                                       conf->xattr_name, 4 * 4);
                 if (ret < 0) {
                         gf_log (this->name, GF_LOG_ERROR,
                                 "Failed to set dict value.");
@@ -222,7 +225,7 @@ nufa_lookup (call_frame_t *frame, xlator_t *this,
         } else {
         do_fresh_lookup:
                 ret = dict_set_uint32 (local->xattr_req,
-                                       "trusted.glusterfs.dht", 4 * 4);
+                                       conf->xattr_name, 4 * 4);
                 if (ret < 0) {
                         gf_log (this->name, GF_LOG_ERROR,
                                 "Failed to set dict value.");
@@ -231,7 +234,7 @@ nufa_lookup (call_frame_t *frame, xlator_t *this,
                 }
 
                 ret = dict_set_uint32 (local->xattr_req,
-                                       "trusted.glusterfs.dht.linkto", 256);
+                                       conf->link_xattr_name, 256);
                 if (ret < 0) {
                         gf_log (this->name, GF_LOG_ERROR,
                                 "Failed to set dict value.");
@@ -320,7 +323,8 @@ nufa_create (call_frame_t *frame, xlator_t *this,
         if (dht_is_subvol_filled (this, (xlator_t *)conf->private)) {
                 avail_subvol =
                         dht_free_disk_available_subvol (this,
-                                                        (xlator_t *)conf->private);
+                                                        (xlator_t *)conf->private,
+                                                        local);
         }
 
         if (subvol != avail_subvol) {
@@ -330,9 +334,8 @@ nufa_create (call_frame_t *frame, xlator_t *this,
                 local->flags = flags;
                 local->umask = umask;
                 local->cached_subvol = avail_subvol;
-                dht_linkfile_create (frame,
-                                     nufa_create_linkfile_create_cbk,
-                                     avail_subvol, subvol, loc);
+                dht_linkfile_create (frame, nufa_create_linkfile_create_cbk,
+                                     this, avail_subvol, subvol, loc);
                 return 0;
         }
 
@@ -362,17 +365,22 @@ nufa_mknod_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         dht_local_t  *local = NULL;
 
         local = frame->local;
+        if (!local || !local->cached_subvol) {
+                op_errno = EINVAL;
+                op_ret = -1;
+                goto err;
+        }
 
         if (op_ret >= 0) {
-                STACK_WIND (frame, dht_newfile_cbk,
-                            local->cached_subvol,
+                STACK_WIND_COOKIE (frame, dht_newfile_cbk,
+                            (void *)local->cached_subvol, local->cached_subvol,
                             local->cached_subvol->fops->mknod,
                             &local->loc, local->mode, local->rdev,
                             local->umask, local->params);
 
                 return 0;
         }
-
+err:
         WIPE (postparent);
         WIPE (preparent);
 
@@ -420,7 +428,8 @@ nufa_mknod (call_frame_t *frame, xlator_t *this,
         if (dht_is_subvol_filled (this, (xlator_t *)conf->private)) {
                 avail_subvol =
                         dht_free_disk_available_subvol (this,
-                                                        (xlator_t *)conf->private);
+                                                        (xlator_t *)conf->private,
+                                                        local);
         }
 
         if (avail_subvol != subvol) {
@@ -432,7 +441,7 @@ nufa_mknod (call_frame_t *frame, xlator_t *this,
                 local->rdev = rdev;
                 local->cached_subvol = avail_subvol;
 
-                dht_linkfile_create (frame, nufa_mknod_linkfile_cbk,
+                dht_linkfile_create (frame, nufa_mknod_linkfile_cbk, this,
                                      avail_subvol, subvol, loc);
                 return 0;
         }
@@ -440,9 +449,9 @@ nufa_mknod (call_frame_t *frame, xlator_t *this,
         gf_log (this->name, GF_LOG_TRACE,
                 "creating %s on %s", loc->path, subvol->name);
 
-        STACK_WIND (frame, dht_newfile_cbk,
-                    subvol, subvol->fops->mknod,
-                    loc, mode, rdev, umask, params);
+        STACK_WIND_COOKIE (frame, dht_newfile_cbk, (void *)subvol, subvol,
+                           subvol->fops->mknod, loc, mode, rdev, umask,
+                           params);
 
         return 0;
 
@@ -455,194 +464,170 @@ err:
 }
 
 
-int
-notify (xlator_t *this, int event, void *data, ...)
+gf_boolean_t
+same_first_part (char *str1, char term1, char *str2, char term2)
 {
-        int ret = -1;
-
-        ret = dht_notify (this, event, data);
-
-        return ret;
+        gf_boolean_t    ended1;
+        gf_boolean_t    ended2;
+
+        for (;;) {
+                ended1 = ((*str1 == '\0') || (*str1 == term1));
+                ended2 = ((*str2 == '\0') || (*str2 == term2));
+                if (ended1 && ended2) {
+                        return _gf_true;
+                }
+                if (ended1 || ended2 || (*str1 != *str2)) {
+                        return _gf_false;
+                }
+                ++str1;
+                ++str2;
+        }
 }
 
-void
-fini (xlator_t *this)
-{
-        int         i = 0;
-        dht_conf_t *conf = NULL;
+typedef struct nufa_args {
+        xlator_t    *this;
+        char        *volname;
+        gf_boolean_t addr_match;
+} nufa_args_t;
 
-        conf = this->private;
+static void
+nufa_find_local_brick (xlator_t *xl, void *data)
+{
+        nufa_args_t     *args = data;
+        xlator_t        *this = args->this;
+        char            *local_volname = args->volname;
+        gf_boolean_t    addr_match = args->addr_match;
+        char            *brick_host = NULL;
+        dht_conf_t      *conf = this->private;
+        int             ret = -1;
+
+        /*This means a local subvol was already found. We pick the first brick
+         * that is local*/
+        if (conf->private)
+                return;
+
+        if (strcmp (xl->name, local_volname) == 0) {
+                conf->private = xl;
+                gf_log (this->name, GF_LOG_INFO, "Using specified subvol %s",
+                        local_volname);
+                return;
+        }
 
-        if (conf) {
-                if (conf->file_layouts) {
-                        for (i = 0; i < conf->subvolume_cnt; i++) {
-                                GF_FREE (conf->file_layouts[i]);
-                        }
-                        GF_FREE (conf->file_layouts);
-                }
+        if (!addr_match)
+                return;
 
-                if (conf->subvolumes)
-                        GF_FREE (conf->subvolumes);
+        ret = dict_get_str (xl->options, "remote-host", &brick_host);
+        if ((ret == 0) &&
+            (gf_is_same_address (local_volname, brick_host) ||
+             gf_is_local_addr (brick_host))) {
+                conf->private = xl;
+                gf_log (this->name, GF_LOG_INFO, "Using the first local "
+                        "subvol %s", xl->name);
+                return;
+        }
 
-                if (conf->subvolume_status)
-                        GF_FREE (conf->subvolume_status);
+}
 
-                GF_FREE (conf);
-        }
+static void
+nufa_to_dht (xlator_t *this)
+{
+        GF_ASSERT (this);
+        GF_ASSERT (this->fops);
 
-        return;
+        this->fops->lookup = dht_lookup;
+        this->fops->create = dht_create;
+        this->fops->mknod  = dht_mknod;
 }
 
 int
-init (xlator_t *this)
+nufa_find_local_subvol (xlator_t *this,
+                        void (*fn) (xlator_t *each, void* data), void *data)
 {
-        dht_conf_t    *conf = NULL;
-        xlator_list_t *trav = NULL;
-        data_t        *data = NULL;
-        char          *local_volname = NULL;
-        char          *temp_str = NULL;
-        int            ret = -1;
-        int            i = 0;
-        char           my_hostname[256];
-        uint32_t       temp_free_disk = 0;
-
-        if (!this->children) {
-                gf_log (this->name, GF_LOG_CRITICAL,
-                        "NUFA needs more than one subvolume");
+        int             ret = -1;
+        dht_conf_t      *conf = this->private;
+        xlator_list_t   *trav = NULL;
+        xlator_t        *parent = NULL;
+        xlator_t        *candidate = NULL;
+
+        xlator_foreach_depth_first (this, fn, data);
+        if (!conf->private) {
+                gf_log (this->name, GF_LOG_ERROR, "Couldn't find a local "
+                        "brick");
                 return -1;
         }
 
-        if (!this->parents) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "dangling volume. check volfile");
-        }
-
-        conf = GF_CALLOC (1, sizeof (*conf),
-                          gf_dht_mt_dht_conf_t);
-        if (!conf) {
-                goto err;
-        }
-
-        conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_ON;
-        if (dict_get_str (this->options, "lookup-unhashed", &temp_str) == 0) {
-                /* If option is not "auto", other options _should_ be boolean */
-                if (strcasecmp (temp_str, "auto"))
-                        gf_string2boolean (temp_str, &conf->search_unhashed);
-                else
-                        conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_AUTO;
-        }
+        candidate = conf->private;
+        trav = candidate->parents;
+        while (trav) {
 
-        ret = dht_init_subvolumes (this, conf);
-        if (ret == -1) {
-                goto err;
-        }
+                parent = trav->xlator;
+                if (strcmp (parent->type, "cluster/nufa") == 0) {
+                        gf_log (this->name, GF_LOG_INFO, "Found local subvol, "
+                                "%s", candidate->name);
+                        ret = 0;
+                        conf->private = candidate;
+                        break;
+                }
 
-        ret = dht_layouts_init (this, conf);
-        if (ret == -1) {
-                goto err;
+                candidate = parent;
+                trav = parent->parents;
         }
 
-        LOCK_INIT (&conf->subvolume_lock);
-        LOCK_INIT (&conf->layout_lock);
+        return ret;
+}
 
-        conf->gen = 1;
+int
+nufa_init (xlator_t *this)
+{
+        data_t        *data = NULL;
+        char          *local_volname = NULL;
+        int            ret = -1;
+        char           my_hostname[256];
+        gf_boolean_t   addr_match = _gf_false;
+        nufa_args_t    args = {0, };
 
-        local_volname = "localhost";
-        ret = gethostname (my_hostname, 256);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "could not find hostname (%s)",
-                        strerror (errno));
+        ret = dht_init(this);
+        if (ret) {
+                return ret;
         }
 
-        if (ret == 0)
-                local_volname = my_hostname;
-
-        data = dict_get (this->options, "local-volume-name");
-        if (data) {
+        if ((data = dict_get (this->options, "local-volume-name"))) {
                 local_volname = data->data;
-        }
 
-        trav = this->children;
-        while (trav) {
-                if (strcmp (trav->xlator->name, local_volname) == 0)
-                        break;
-                trav = trav->next;
-        }
+        } else {
+                addr_match = _gf_true;
+                local_volname = "localhost";
+                ret = gethostname (my_hostname, 256);
+                if (ret == 0)
+                        local_volname = my_hostname;
 
-        if (!trav) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Could not find subvolume named '%s'. "
-                        "Please define volume with the name as the hostname "
-                        "or override it with 'option local-volume-name'",
-                        local_volname);
-                goto err;
-        }
-        /* The volume specified exists */
-        conf->private = trav->xlator;
-
-        conf->min_free_disk = 10;
-        conf->disk_unit = 'p';
-
-        if (dict_get_str (this->options, "min-free-disk",
-                          &temp_str) == 0) {
-                if (gf_string2percent (temp_str,
-                                       &temp_free_disk) == 0) {
-                        if (temp_free_disk > 100) {
-                                gf_string2bytesize (temp_str,
-                                                    &conf->min_free_disk);
-                                conf->disk_unit = 'b';
-                        } else {
-                                conf->min_free_disk = (uint64_t)temp_free_disk;
-                                conf->disk_unit = 'p';
-                        }
-                } else {
-                        gf_string2bytesize (temp_str,
-                                            &conf->min_free_disk);
-                        conf->disk_unit = 'b';
-                }
-        }
+                else
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "could not find hostname (%s)",
+                                strerror (errno));
 
-        conf->du_stats = GF_CALLOC (conf->subvolume_cnt, sizeof (dht_du_t),
-                                    gf_dht_mt_dht_du_t);
-        if (!conf->du_stats) {
-                goto err;
         }
 
-        this->local_pool = mem_pool_new (dht_local_t, 128);
-        if (!this->local_pool) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to create local_t's memory pool");
-                goto err;
+        args.this = this;
+        args.volname = local_volname;
+        args.addr_match = addr_match;
+        ret = nufa_find_local_subvol (this, nufa_find_local_brick, &args);
+        if (ret) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "Unable to find local subvolume, switching "
+                        "to dht mode");
+                nufa_to_dht (this);
         }
-
-        this->private = conf;
-
         return 0;
+}
 
-err:
-        if (conf) {
-                if (conf->file_layouts) {
-                        for (i = 0; i < conf->subvolume_cnt; i++) {
-                                GF_FREE (conf->file_layouts[i]);
-                        }
-                        GF_FREE (conf->file_layouts);
-                }
-
-                if (conf->subvolumes)
-                        GF_FREE (conf->subvolumes);
-
-                if (conf->subvolume_status)
-                        GF_FREE (conf->subvolume_status);
-
-                if (conf->du_stats)
-                        GF_FREE (conf->du_stats);
 
-                GF_FREE (conf);
-        }
-
-        return -1;
-}
+class_methods_t class_methods = {
+        .init           = nufa_init,
+        .fini           = dht_fini,
+        .reconfigure    = dht_reconfigure,
+        .notify         = dht_notify
+};
 
 
 struct xlator_fops fops = {
@@ -689,19 +674,3 @@ struct xlator_fops fops = {
 struct xlator_cbks cbks = {
         .forget     = dht_forget
 };
-
-
-struct volume_options options[] = {
-        { .key  = {"lookup-unhashed"},
-          .value = {"auto", "yes", "no", "enable", "disable", "1", "0",
-                    "on", "off"},
-          .type = GF_OPTION_TYPE_STR
-        },
-        { .key  = {"local-volume-name"},
-          .type = GF_OPTION_TYPE_XLATOR
-        },
-        { .key  = {"min-free-disk"},
-          .type = GF_OPTION_TYPE_PERCENT_OR_SIZET,
-        },
-        { .key  = {NULL} },
-};
diff --git a/xlators/cluster/dht/src/switch.c b/xlators/cluster/dht/src/switch.c
index fe75914f2..2717ce975 100644
--- a/xlators/cluster/dht/src/switch.c
+++ b/xlators/cluster/dht/src/switch.c
@@ -22,6 +22,8 @@
 #include <fnmatch.h>
 #include <string.h>
 
+extern struct volume_options options[];
+
 struct switch_sched_array {
         xlator_t *xl;
         int32_t   eligible;
@@ -67,29 +69,37 @@ get_switch_matching_subvol (const char *path, dht_conf_t *conf,
         struct switch_struct *cond      = NULL;
         struct switch_struct *trav      = NULL;
         char                 *pathname  = NULL;
-        int                   idx     = 0;
+        int                   idx       = 0;
+        xlator_t             *subvol    = NULL;
 
         cond = conf->private;
+        subvol = hashed_subvol;
         if (!cond)
-                return hashed_subvol;
+                goto out;
 
-        trav = cond;
         pathname = gf_strdup (path);
+        if (!pathname)
+                goto out;
+
+        trav = cond;
         while (trav) {
                 if (fnmatch (trav->path_pattern,
                              pathname, FNM_NOESCAPE) == 0) {
                         for (idx = 0; idx < trav->num_child; idx++) {
                                 if (trav->array[idx].xl == hashed_subvol)
-                                        return hashed_subvol;
+                                        goto out;
                         }
                         idx = trav->node_index++;
                         trav->node_index %= trav->num_child;
-                        return trav->array[idx].xl;
+                        subvol = trav->array[idx].xl;
+                        goto out;
                 }
                 trav = trav->next;
         }
+out:
         GF_FREE (pathname);
-        return hashed_subvol;
+
+        return subvol;
 }
 
 
@@ -127,7 +137,8 @@ switch_local_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if (op_ret == -1)
                 goto out;
 
-        is_linkfile = check_is_linkfile (inode, stbuf, xattr);
+        is_linkfile = check_is_linkfile (inode, stbuf, xattr,
+                                         conf->link_xattr_name);
         is_dir      = check_is_dir (inode, stbuf, xattr);
 
         if (!is_dir && !is_linkfile) {
@@ -281,11 +292,11 @@ switch_lookup (call_frame_t *frame, xlator_t *this,
                  * attribute, revalidates directly go to the cached-subvolume.
                  */
                 ret = dict_set_uint32 (local->xattr_req,
-                                       "trusted.glusterfs.dht", 4 * 4);
+                                       conf->xattr_name, 4 * 4);
                 if (ret < 0)
                         gf_log (this->name, GF_LOG_WARNING,
-                                "failed to set dict value for "
-                                "trusted.glusterfs.dht");
+                                "failed to set dict value for %s",
+                                conf->xattr_name);
 
                 for (i = 0; i < layout->cnt; i++) {
                         subvol = layout->list[i].xlator;
@@ -300,18 +311,18 @@ switch_lookup (call_frame_t *frame, xlator_t *this,
         } else {
         do_fresh_lookup:
                 ret = dict_set_uint32 (local->xattr_req,
-                                       "trusted.glusterfs.dht", 4 * 4);
+                                       conf->xattr_name, 4 * 4);
                 if (ret < 0)
                         gf_log (this->name, GF_LOG_WARNING,
-                                "failed to set dict value for "
-                                "trusted.glusterfs.dht");
+                                "failed to set dict value for %s",
+                                conf->xattr_name);
 
                 ret = dict_set_uint32 (local->xattr_req,
-                                       "trusted.glusterfs.dht.linkto", 256);
+                                       conf->link_xattr_name, 256);
                 if (ret < 0)
                         gf_log (this->name, GF_LOG_WARNING,
-                                "failed to set dict value for "
-                                "trusted.glusterfs.dht.linkto");
+                                "failed to set dict value for %s",
+                                conf->link_xattr_name);
 
                 if (!hashed_subvol) {
                         gf_log (this->name, GF_LOG_DEBUG,
@@ -426,7 +437,8 @@ switch_create (call_frame_t *frame, xlator_t *this,
         avail_subvol = get_switch_matching_subvol (loc->path, conf, subvol);
         if (dht_is_subvol_filled (this, avail_subvol)) {
                 avail_subvol =
-                        dht_free_disk_available_subvol (this, avail_subvol);
+                        dht_free_disk_available_subvol (this, avail_subvol,
+                                                        local);
         }
 
         if (subvol != avail_subvol) {
@@ -435,9 +447,8 @@ switch_create (call_frame_t *frame, xlator_t *this,
                 local->flags = flags;
                 local->umask = umask;
                 local->cached_subvol = avail_subvol;
-                dht_linkfile_create (frame,
-                                     switch_create_linkfile_create_cbk,
-                                     avail_subvol, subvol, loc);
+                dht_linkfile_create (frame, switch_create_linkfile_create_cbk,
+                                     this, avail_subvol, subvol, loc);
                 return 0;
         }
 
@@ -467,17 +478,22 @@ switch_mknod_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         dht_local_t  *local = NULL;
 
         local = frame->local;
+        if (!local || !local->cached_subvol) {
+                op_errno = EINVAL;
+                op_ret = -1;
+                goto err;
+        }
 
         if (op_ret >= 0) {
-                STACK_WIND (frame, dht_newfile_cbk,
-                            local->cached_subvol,
+                STACK_WIND_COOKIE (frame, dht_newfile_cbk,
+                            (void *)local->cached_subvol, local->cached_subvol,
                             local->cached_subvol->fops->mknod,
                             &local->loc, local->mode, local->rdev,
                             local->umask, local->params);
 
                 return 0;
         }
-
+err:
         DHT_STACK_UNWIND (link, frame, op_ret, op_errno,
                           inode, stbuf, preparent, postparent, xdata);
         return 0;
@@ -521,7 +537,8 @@ switch_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
         avail_subvol = get_switch_matching_subvol (loc->path, conf, subvol);
         if (dht_is_subvol_filled (this, avail_subvol)) {
                 avail_subvol =
-                        dht_free_disk_available_subvol (this, avail_subvol);
+                        dht_free_disk_available_subvol (this, avail_subvol,
+                                                        local);
         }
 
         if (avail_subvol != subvol) {
@@ -534,16 +551,16 @@ switch_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
                 local->cached_subvol = avail_subvol;
 
                 dht_linkfile_create (frame, switch_mknod_linkfile_cbk,
-                                     avail_subvol, subvol, loc);
+                                     this, avail_subvol, subvol, loc);
                 return 0;
         }
 
         gf_log (this->name, GF_LOG_TRACE,
                 "creating %s on %s", loc->path, subvol->name);
 
-        STACK_WIND (frame, dht_newfile_cbk,
-                    subvol, subvol->fops->mknod,
-                    loc, mode, rdev, umask, params);
+        STACK_WIND_COOKIE (frame, dht_newfile_cbk, (void *)subvol, subvol,
+                           subvol->fops->mknod, loc, mode, rdev, umask,
+                           params);
 
         return 0;
 
@@ -556,20 +573,9 @@ err:
 }
 
 
-int
-notify (xlator_t *this, int event, void *data, ...)
-{
-        int ret = -1;
-
-        ret = dht_notify (this, event, data);
-
-        return ret;
-}
-
 void
-fini (xlator_t *this)
+switch_fini (xlator_t *this)
 {
-        int                   i = 0;
         dht_conf_t           *conf = NULL;
         struct switch_struct *trav = NULL;
         struct switch_struct *prev = NULL;
@@ -580,30 +586,14 @@ fini (xlator_t *this)
                 trav = (struct switch_struct *)conf->private;
                 conf->private = NULL;
                 while (trav) {
-                        if (trav->array)
-                                GF_FREE (trav->array);
+                        GF_FREE (trav->array);
                         prev = trav;
                         trav = trav->next;
                         GF_FREE (prev);
                 }
-
-                if (conf->file_layouts) {
-                        for (i = 0; i < conf->subvolume_cnt; i++) {
-                                GF_FREE (conf->file_layouts[i]);
-                        }
-                        GF_FREE (conf->file_layouts);
-                }
-
-                if (conf->subvolumes)
-                        GF_FREE (conf->subvolumes);
-
-                if (conf->subvolume_status)
-                        GF_FREE (conf->subvolume_status);
-
-                GF_FREE (conf);
         }
 
-        return;
+        dht_fini(this);
 }
 
 int
@@ -663,8 +653,10 @@ set_switch_pattern (xlator_t *this, dht_conf_t *conf,
                 dup_str = gf_strdup (switch_str);
                 switch_opt = GF_CALLOC (1, sizeof (struct switch_struct),
                                         gf_switch_mt_switch_struct);
-                if (!switch_opt)
+                if (!switch_opt) {
+                        GF_FREE (dup_str);
                         goto err;
+                }
 
                 pattern = strtok_r (dup_str, ":", &tmp_str1);
                 childs = strtok_r (NULL, ":", &tmp_str1);
@@ -674,9 +666,11 @@ set_switch_pattern (xlator_t *this, dht_conf_t *conf,
                                 "for all the unconfigured child nodes,"
                                 " hence neglecting current option");
                         switch_str = strtok_r (NULL, ";", &tmp_str);
+                        GF_FREE (switch_opt);
                         GF_FREE (dup_str);
                         continue;
                 }
+                GF_FREE (dup_str);
                 memcpy (switch_opt->path_pattern, pattern, strlen (pattern));
                 if (childs) {
                         dup_childs = gf_strdup (childs);
@@ -733,7 +727,6 @@ set_switch_pattern (xlator_t *this, dht_conf_t *conf,
                                 "option in unify volume. Exiting");
                         goto err;
                 }
-                GF_FREE (dup_str);
 
                 /* Link it to the main structure */
                 if (switch_buf) {
@@ -746,6 +739,7 @@ set_switch_pattern (xlator_t *this, dht_conf_t *conf,
                         /* First entry */
                         switch_buf = switch_opt;
                 }
+                switch_opt = NULL;
                 switch_str = strtok_r (NULL, ";", &tmp_str);
         }
 
@@ -802,19 +796,20 @@ set_switch_pattern (xlator_t *this, dht_conf_t *conf,
                         /* First entry */
                         switch_buf = switch_opt;
                 }
+                switch_opt = NULL;
         }
         /* */
         conf->private = switch_buf;
 
         return 0;
 err:
+        GF_FREE (switch_buf_array);
+        GF_FREE (switch_opt);
+
         if (switch_buf) {
-                if (switch_buf_array)
-                        GF_FREE (switch_buf_array);
                 trav = switch_buf;
                 while (trav) {
-                        if (trav->array)
-                                GF_FREE (trav->array);
+                        GF_FREE (trav->array);
                         switch_opt = trav;
                         trav = trav->next;
                         GF_FREE (switch_opt);
@@ -824,68 +819,18 @@ err:
 }
 
 
-int
-init (xlator_t *this)
+int32_t
+switch_init (xlator_t *this)
 {
         dht_conf_t            *conf = NULL;
         data_t                *data = NULL;
-        char                  *temp_str = NULL;
         int                    ret = -1;
-        int                    i = 0;
-        uint32_t               temp_free_disk = 0;
-
-        if (!this->children) {
-                gf_log (this->name, GF_LOG_CRITICAL,
-                        "SWITCH needs more than one subvolume");
-                return -1;
-        }
-
-        if (!this->parents) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "dangling volume. check volfile");
-        }
-
-        conf = GF_CALLOC (1, sizeof (*conf), gf_switch_mt_dht_conf_t);
-        if (!conf) {
-                goto err;
-        }
-
-        conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_ON;
-        if (dict_get_str (this->options, "lookup-unhashed", &temp_str) == 0) {
-                /* If option is not "auto", other options _should_ be boolean */
-                if (strcasecmp (temp_str, "auto"))
-                        gf_string2boolean (temp_str, &conf->search_unhashed);
-                else
-                        conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_AUTO;
-        }
-
-        conf->unhashed_sticky_bit = 0;
-        if (dict_get_str (this->options, "unhashed-sticky-bit",
-                          &temp_str) == 0) {
-                gf_string2boolean (temp_str, &conf->unhashed_sticky_bit);
-        }
 
-        conf->min_free_disk = 10;
-        conf->disk_unit = 'p';
-
-        if (dict_get_str (this->options, "min-free-disk",
-                          &temp_str) == 0) {
-                if (gf_string2percent (temp_str,
-                                       &temp_free_disk) == 0) {
-                        if (temp_free_disk > 100) {
-                                gf_string2bytesize (temp_str,
-                                                    &conf->min_free_disk);
-                                conf->disk_unit = 'b';
-                        } else {
-                                conf->min_free_disk = (uint64_t)temp_free_disk;
-                                conf->disk_unit = 'p';
-                        }
-                } else {
-                        gf_string2bytesize (temp_str,
-                                            &conf->min_free_disk);
-                        conf->disk_unit = 'b';
-                }
+        ret = dht_init(this);
+        if (ret) {
+                return ret;
         }
+        conf = this->private;
 
         data = dict_get (this->options, "pattern.switch.case");
         if (data) {
@@ -896,63 +841,23 @@ init (xlator_t *this)
                 }
         }
 
-        ret = dht_init_subvolumes (this, conf);
-        if (ret == -1) {
-                goto err;
-        }
-
-        ret = dht_layouts_init (this, conf);
-        if (ret == -1) {
-                goto err;
-        }
-
-        LOCK_INIT (&conf->subvolume_lock);
-        LOCK_INIT (&conf->layout_lock);
-
-        conf->gen = 1;
-
-        conf->du_stats = GF_CALLOC (conf->subvolume_cnt, sizeof (dht_du_t),
-                                    gf_switch_mt_dht_du_t);
-        if (!conf->du_stats) {
-                goto err;
-        }
-
-        this->local_pool = mem_pool_new (dht_local_t, 128);
-        if (!this->local_pool) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to create local_t's memory pool");
-                goto err;
-        }
-
         this->private = conf;
-
         return 0;
 
 err:
-        if (conf) {
-                if (conf->file_layouts) {
-                        for (i = 0; i < conf->subvolume_cnt; i++) {
-                                GF_FREE (conf->file_layouts[i]);
-                        }
-                        GF_FREE (conf->file_layouts);
-                }
-
-                if (conf->subvolumes)
-                        GF_FREE (conf->subvolumes);
-
-                if (conf->subvolume_status)
-                        GF_FREE (conf->subvolume_status);
-
-                if (conf->du_stats)
-                        GF_FREE (conf->du_stats);
-
-                GF_FREE (conf);
-        }
-
+        dht_fini(this);
         return -1;
 }
 
 
+class_methods_t class_methods = {
+        .init           = switch_init,
+        .fini           = switch_fini,
+        .reconfigure    = dht_reconfigure,
+        .notify         = dht_notify
+};
+
+
 struct xlator_fops fops = {
         .lookup      = switch_lookup,
         .create      = switch_create,
@@ -997,19 +902,3 @@ struct xlator_fops fops = {
 struct xlator_cbks cbks = {
         .forget     = dht_forget
 };
-
-
-struct volume_options options[] = {
-        { .key  = {"lookup-unhashed"},
-          .value = {"auto", "yes", "no", "enable", "disable", "1", "0",
-                    "on", "off"},
-          .type = GF_OPTION_TYPE_STR
-        },
-        { .key  = {"pattern.switch.case"},
-          .type = GF_OPTION_TYPE_ANY
-        },
-        { .key  = {"min-free-disk"},
-          .type = GF_OPTION_TYPE_PERCENT_OR_SIZET,
-        },
-        { .key  = {NULL} },
-};
diff --git a/xlators/cluster/dht/src/unittest/dht_layout_mock.c b/xlators/cluster/dht/src/unittest/dht_layout_mock.c
new file mode 100644
index 000000000..aa19ddc57
--- /dev/null
+++ b/xlators/cluster/dht/src/unittest/dht_layout_mock.c
@@ -0,0 +1,63 @@
+/*
+  Copyright (c) 2014 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "xlator.h"
+#include "dht-common.h"
+#include "byte-order.h"
+
+int
+dht_hash_compute (xlator_t *this, int type, const char *name, uint32_t *hash_p)
+{
+    return 0;
+}
+
+int
+dht_inode_ctx_layout_get (inode_t *inode, xlator_t *this, dht_layout_t **layout)
+{
+    return 0;
+}
+
+int
+dht_inode_ctx_layout_set (inode_t *inode, xlator_t *this,
+                          dht_layout_t *layout_int)
+{
+    return 0;
+}
+
+int
+dict_get_ptr (dict_t *this, char *key, void **ptr)
+{
+    return 0;
+}
+
+int
+dict_get_ptr_and_len (dict_t *this, char *key, void **ptr, int *len)
+{
+    return 0;
+}
+
+int _gf_log (const char *domain, const char *file,
+             const char *function, int32_t line, gf_loglevel_t level,
+             const char *fmt, ...)
+{
+    return 0;
+}
+
+int _gf_log_callingfn (const char *domain, const char *file,
+                       const char *function, int32_t line, gf_loglevel_t level,
+                       const char *fmt, ...)
+{
+    return 0;
+}
diff --git a/xlators/cluster/dht/src/unittest/dht_layout_unittest.c b/xlators/cluster/dht/src/unittest/dht_layout_unittest.c
new file mode 100644
index 000000000..b5233d235
--- /dev/null
+++ b/xlators/cluster/dht/src/unittest/dht_layout_unittest.c
@@ -0,0 +1,124 @@
+/*
+  Copyright (c) 2008-2014 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include "dht-common.h"
+#include "logging.h"
+#include "xlator.h"
+
+#include <stdarg.h>
+#include <stddef.h>
+#include <setjmp.h>
+#include <inttypes.h>
+#include <cmockery/pbc.h>
+#include <cmockery/cmockery.h>
+
+/*
+ * Helper functions
+ */
+
+static xlator_t *
+helper_xlator_init(uint32_t num_types)
+{
+    xlator_t *xl;
+    int i, ret;
+
+    REQUIRE(num_types > 0);
+
+    xl = test_calloc(1, sizeof(xlator_t));
+    assert_non_null(xl);
+    xl->mem_acct.num_types = num_types;
+    xl->mem_acct.rec = test_calloc(num_types, sizeof(struct mem_acct_rec));
+    assert_non_null(xl->mem_acct.rec);
+
+    xl->ctx = test_calloc(1, sizeof(glusterfs_ctx_t));
+    assert_non_null(xl->ctx);
+
+    for (i = 0; i < num_types; i++) {
+            ret = LOCK_INIT(&(xl->mem_acct.rec[i].lock));
+            assert_false(ret);
+    }
+
+    ENSURE(num_types == xl->mem_acct.num_types);
+    ENSURE(NULL != xl);
+
+    return xl;
+}
+
+static int
+helper_xlator_destroy(xlator_t *xl)
+{
+    int i, ret;
+
+    for (i = 0; i < xl->mem_acct.num_types; i++) {
+            ret = LOCK_DESTROY(&(xl->mem_acct.rec[i].lock));
+            assert_int_equal(ret, 0);
+    }
+
+    free(xl->mem_acct.rec);
+    free(xl->ctx);
+    free(xl);
+    return 0;
+}
+
+/*
+ * Unit tests
+ */
+static void
+test_dht_layout_new(void **state)
+{
+    xlator_t *xl;
+    dht_layout_t *layout;
+    dht_conf_t   *conf;
+    int cnt;
+
+    expect_assert_failure(dht_layout_new(NULL, 0));
+    expect_assert_failure(dht_layout_new((xlator_t *)0x12345, -1));
+    xl = helper_xlator_init(10);
+
+    // xl->private is NULL
+    assert_null(xl->private);
+    cnt = 100;
+    layout = dht_layout_new(xl, cnt);
+    assert_non_null(layout);
+    assert_int_equal(layout->type, DHT_HASH_TYPE_DM);
+    assert_int_equal(layout->cnt, cnt);
+    assert_int_equal(layout->ref, 1);
+    assert_int_equal(layout->gen, 0);
+    assert_int_equal(layout->spread_cnt, 0);
+    free(layout);
+
+    // xl->private is not NULL
+    cnt = 110;
+    conf = (dht_conf_t *)test_calloc(1, sizeof(dht_conf_t));
+    assert_non_null(conf);
+    conf->dir_spread_cnt = 12345;
+    conf->gen = -123;
+    xl->private = conf;
+
+    layout = dht_layout_new(xl, cnt);
+    assert_non_null(layout);
+    assert_int_equal(layout->type, DHT_HASH_TYPE_DM);
+    assert_int_equal(layout->cnt, cnt);
+    assert_int_equal(layout->ref, 1);
+    assert_int_equal(layout->gen, conf->gen);
+    assert_int_equal(layout->spread_cnt, conf->dir_spread_cnt);
+    free(layout);
+
+    free(conf);
+    helper_xlator_destroy(xl);
+}
+
+int main(void) {
+    const UnitTest tests[] = {
+        unit_test(test_dht_layout_new),
+    };
+
+    return run_tests(tests, "xlator_dht_layout");
+}
diff --git a/xlators/cluster/ha/src/Makefile.am b/xlators/cluster/ha/src/Makefile.am
index 5f78a2965..5c1364b7f 100644
--- a/xlators/cluster/ha/src/Makefile.am
+++ b/xlators/cluster/ha/src/Makefile.am
@@ -1,15 +1,16 @@
 xlator_LTLIBRARIES = ha.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/testing/cluster
 
-ha_la_LDFLAGS = -module -avoidversion 
+ha_la_LDFLAGS = -module -avoid-version 
 
 ha_la_SOURCES = ha-helpers.c ha.c
 ha_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = ha.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
-	    -I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES = 
 
diff --git a/xlators/cluster/ha/src/ha-helpers.c b/xlators/cluster/ha/src/ha-helpers.c
index 1e4af1b62..19be1ed27 100644
--- a/xlators/cluster/ha/src/ha-helpers.c
+++ b/xlators/cluster/ha/src/ha-helpers.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #include "xlator.h"
 #include "call-stub.h"
 #include "defaults.h"
diff --git a/xlators/cluster/ha/src/ha-mem-types.h b/xlators/cluster/ha/src/ha-mem-types.h
index 9bfb3972b..e5e97d237 100644
--- a/xlators/cluster/ha/src/ha-mem-types.h
+++ b/xlators/cluster/ha/src/ha-mem-types.h
@@ -1,24 +1,13 @@
-
 /*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef __HA_MEM_TYPES_H__
 #define __HA_MEM_TYPES_H__
 
diff --git a/xlators/cluster/ha/src/ha.c b/xlators/cluster/ha/src/ha.c
index 38d4229d3..3eccb516b 100644
--- a/xlators/cluster/ha/src/ha.c
+++ b/xlators/cluster/ha/src/ha.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 /* generate errors randomly, code is simple now, better alogorithm
  * can be written to decide what error to be returned and when
  */
@@ -1876,13 +1866,9 @@ err:
         }
 
         if (hafdp) {
-                if (hafdp->fdstate) {
-                        GF_FREE (hafdp->fdstate);
-                }
+                GF_FREE (hafdp->fdstate);
 
-                if (hafdp->path) {
-                        GF_FREE (hafdp->path);
-                }
+                GF_FREE (hafdp->path);
 
                 GF_FREE (hafdp);
         }
diff --git a/xlators/cluster/ha/src/ha.h b/xlators/cluster/ha/src/ha.h
index 39b6851e7..e2ed7eaa6 100644
--- a/xlators/cluster/ha/src/ha.h
+++ b/xlators/cluster/ha/src/ha.h
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef __HA_H_
 #define __HA_H_
 
diff --git a/xlators/cluster/map/src/Makefile.am b/xlators/cluster/map/src/Makefile.am
index 26e19137a..a278b05e2 100644
--- a/xlators/cluster/map/src/Makefile.am
+++ b/xlators/cluster/map/src/Makefile.am
@@ -1,15 +1,16 @@
 xlator_LTLIBRARIES = map.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/testing/cluster
 
-map_la_LDFLAGS = -module -avoidversion 
+map_la_LDFLAGS = -module -avoid-version 
 
 map_la_SOURCES = map.c map-helper.c
 map_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = map.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
-	    -I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES = 
 
diff --git a/xlators/cluster/map/src/map-helper.c b/xlators/cluster/map/src/map-helper.c
index 81212fcfd..851397b68 100644
--- a/xlators/cluster/map/src/map-helper.c
+++ b/xlators/cluster/map/src/map-helper.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2009-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2009-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
diff --git a/xlators/cluster/map/src/map-mem-types.h b/xlators/cluster/map/src/map-mem-types.h
index 669b93dc2..3e89f4736 100644
--- a/xlators/cluster/map/src/map-mem-types.h
+++ b/xlators/cluster/map/src/map-mem-types.h
@@ -1,24 +1,13 @@
-
 /*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef __MAP_MEM_TYPES_H__
 #define __MAP_MEM_TYPES_H__
 
diff --git a/xlators/cluster/map/src/map.c b/xlators/cluster/map/src/map.c
index ead9da0b9..6150a33ce 100644
--- a/xlators/cluster/map/src/map.c
+++ b/xlators/cluster/map/src/map.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -2375,8 +2365,7 @@ fini (xlator_t *this)
 	priv = this->private;
 
 	if (priv) {
-		if (priv->xlarray)
-			GF_FREE (priv->xlarray);
+		GF_FREE (priv->xlarray);
 
 		trav_map = priv->map;
 		while (trav_map) {
diff --git a/xlators/cluster/map/src/map.h b/xlators/cluster/map/src/map.h
index bccac437c..7703a543e 100644
--- a/xlators/cluster/map/src/map.h
+++ b/xlators/cluster/map/src/map.h
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef __MAP_H__
 #define __MAP_H__
 
diff --git a/xlators/cluster/stripe/src/Makefile.am b/xlators/cluster/stripe/src/Makefile.am
index 8c48d3410..2d151422a 100644
--- a/xlators/cluster/stripe/src/Makefile.am
+++ b/xlators/cluster/stripe/src/Makefile.am
@@ -2,7 +2,7 @@
 xlator_LTLIBRARIES = stripe.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/cluster
 
-stripe_la_LDFLAGS = -module -avoidversion
+stripe_la_LDFLAGS = -module -avoid-version
 
 stripe_la_SOURCES = stripe.c stripe-helpers.c \
 	$(top_builddir)/xlators/lib/src/libxlator.c
@@ -11,9 +11,10 @@ stripe_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = stripe.h stripe-mem-types.h $(top_builddir)/xlators/lib/src/libxlator.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)\
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS) \
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
 	-I$(top_srcdir)/xlators/lib/src
 
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
 CLEANFILES = 
 
diff --git a/xlators/cluster/stripe/src/stripe-helpers.c b/xlators/cluster/stripe/src/stripe-helpers.c
index 3e689413b..a047d4a2e 100644
--- a/xlators/cluster/stripe/src/stripe-helpers.c
+++ b/xlators/cluster/stripe/src/stripe-helpers.c
@@ -12,6 +12,7 @@
 
 #include "stripe.h"
 #include "byte-order.h"
+#include "mem-types.h"
 
 void
 stripe_local_wipe (stripe_local_t *local)
@@ -40,7 +41,7 @@ out:
 
 
 
-void
+int
 stripe_aggregate (dict_t *this, char *key, data_t *value, void *data)
 {
         dict_t  *dst  = NULL;
@@ -83,7 +84,7 @@ stripe_aggregate (dict_t *this, char *key, data_t *value, void *data)
         }
 
 out:
-        return;
+        return 0;
 }
 
 
@@ -158,6 +159,93 @@ stripe_free_xattr_str (stripe_local_t *local)
 
 
 int32_t
+stripe_fill_lockinfo_xattr (xlator_t *this, stripe_local_t *local,
+                            void **xattr_serz)
+{
+        int32_t              ret   = -1, i = 0, len = 0;
+        dict_t              *tmp1  = NULL, *tmp2 = NULL;
+        char                *buf   = NULL;
+        stripe_xattr_sort_t *xattr = NULL;
+
+        if (xattr_serz == NULL) {
+                goto out;
+        }
+
+        tmp2 = dict_new ();
+
+        if (tmp2 == NULL) {
+                goto out;
+        }
+
+        for (i = 0; i < local->nallocs; i++) {
+                xattr = local->xattr_list + i;
+                len = xattr->xattr_len;
+
+                if (len && xattr && xattr->xattr_value) {
+                        ret = dict_reset (tmp2);
+                        if (ret < 0) {
+                                gf_log (this->name, GF_LOG_DEBUG,
+                                        "dict_reset failed (%s)",
+                                        strerror (-ret));
+                        }
+
+                        ret = dict_unserialize (xattr->xattr_value,
+                                                xattr->xattr_len,
+                                                &tmp2);
+                        if (ret < 0) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "dict_unserialize failed (%s)",
+                                        strerror (-ret));
+                                ret = -1;
+                                goto out;
+                        }
+
+                        tmp1 = dict_copy (tmp2, tmp1);
+                        if (tmp1 == NULL) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "dict_copy failed (%s)",
+                                        strerror (-ret));
+                                ret = -1;
+                                goto out;
+                        }
+                }
+        }
+
+        len = dict_serialized_length (tmp1);
+        if (len > 0) {
+                buf = GF_CALLOC (1, len, gf_common_mt_dict_t);
+                if (buf == NULL) {
+                        ret = -1;
+                        goto out;
+                }
+
+                ret = dict_serialize (tmp1, buf);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "dict_serialize failed (%s)", strerror (-ret));
+                        GF_FREE(buf);
+                        ret = -1;
+                        goto out;
+                }
+
+                *xattr_serz = buf;
+        }
+
+        ret = 0;
+out:
+        if (tmp1 != NULL) {
+                dict_unref (tmp1);
+        }
+
+        if (tmp2 != NULL) {
+                dict_unref (tmp2);
+        }
+
+        return ret;
+}
+
+
+int32_t
 stripe_fill_pathinfo_xattr (xlator_t *this, stripe_local_t *local,
                             char **xattr_serz)
 {
@@ -193,6 +281,7 @@ stripe_fill_pathinfo_xattr (xlator_t *this, stripe_local_t *local,
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "Cannot aggregate pathinfo list");
+                GF_FREE(pathinfo_serz);
                 goto out;
         }
 
@@ -493,19 +582,24 @@ set_stripe_block_size (xlator_t *this, stripe_private_t *priv, char *data)
                         temp_stripeopt = NULL;
                 else
                         temp_stripeopt = priv->pattern;
-                priv->pattern = stripe_opt;
+
                 stripe_opt->next = temp_stripeopt;
 
-                stripe_str = strtok_r (NULL, ",", &tmp_str);
+                priv->pattern = stripe_opt;
+                stripe_opt = NULL;
+
                 GF_FREE (dup_str);
                 dup_str = NULL;
+
+                stripe_str = strtok_r (NULL, ",", &tmp_str);
         }
 
         ret = 0;
 out:
 
-        if (dup_str)
-                GF_FREE (dup_str);
+        GF_FREE (dup_str);
+
+        GF_FREE (stripe_opt);
 
         return ret;
 }
diff --git a/xlators/cluster/stripe/src/stripe-mem-types.h b/xlators/cluster/stripe/src/stripe-mem-types.h
index c8781d7d7..e9ac9cf46 100644
--- a/xlators/cluster/stripe/src/stripe-mem-types.h
+++ b/xlators/cluster/stripe/src/stripe-mem-types.h
@@ -16,10 +16,11 @@
 
 enum gf_stripe_mem_types_ {
         gf_stripe_mt_iovec = gf_common_mt_end + 1,
-        gf_stripe_mt_readv_replies,
+        gf_stripe_mt_stripe_replies,
         gf_stripe_mt_stripe_fd_ctx_t,
         gf_stripe_mt_char,
         gf_stripe_mt_int8_t,
+        gf_stripe_mt_int32_t,
         gf_stripe_mt_xlator_t,
         gf_stripe_mt_stripe_private_t,
         gf_stripe_mt_stripe_options,
diff --git a/xlators/cluster/stripe/src/stripe.c b/xlators/cluster/stripe/src/stripe.c
index d836d6392..79e80b513 100644
--- a/xlators/cluster/stripe/src/stripe.c
+++ b/xlators/cluster/stripe/src/stripe.c
@@ -1053,6 +1053,9 @@ stripe_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
                 op_errno = ENOMEM;
                 goto err;
         }
+
+        frame->local = local;
+
         local->op_ret = -1;
         loc_copy (&local->loc, oldloc);
         loc_copy (&local->loc2, newloc);
@@ -1066,8 +1069,6 @@ stripe_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
 		local->fctx = fctx;
 	}
 
-        frame->local = local;
-
         STACK_WIND (frame, stripe_first_rename_cbk, trav->xlator,
                     trav->xlator->fops->rename, oldloc, newloc, NULL);
 
@@ -2879,15 +2880,15 @@ stripe_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags, dict
                 goto err;
         }
 
+        frame->local = local;
+
 	inode_ctx_get(fd->inode, this, (uint64_t *) &fctx);
 	if (!fctx) {
 		op_errno = EINVAL;
 		goto err;
 	}
 	local->fctx = fctx;
-
         local->op_ret = -1;
-        frame->local = local;
         local->call_count = priv->child_count;
 
         while (trav) {
@@ -3244,9 +3245,11 @@ stripe_readv_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                         goto done;
                                 }
                                 memset (iobuf->ptr, 0, vec[count].iov_len);
-                                iobref_add (local->iobref, iobuf);
                                 vec[count].iov_base = iobuf->ptr;
 
+                                iobref_add (local->iobref, iobuf);
+                                iobuf_unref(iobuf);
+
                                 op_ret += vec[count].iov_len;
                                 count++;
                         }
@@ -3267,8 +3270,7 @@ stripe_readv_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                      count, &tmp_stbuf, tmp_iobref, NULL);
 
                 iobref_unref (tmp_iobref);
-                if (vec)
-                        GF_FREE (vec);
+                GF_FREE (vec);
         }
 out:
         return 0;
@@ -3401,8 +3403,7 @@ stripe_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                      final_count, &tmp_stbuf, tmp_iobref, NULL);
 
                 iobref_unref (tmp_iobref);
-                if (final_vec)
-                        GF_FREE (final_vec);
+                GF_FREE (final_vec);
         }
 
         goto out;
@@ -3482,8 +3483,8 @@ stripe_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
         frame->local = local;
 
         /* This is where all the vectors should be copied. */
-        local->replies = GF_CALLOC (num_stripe, sizeof (struct readv_replies),
-                                    gf_stripe_mt_readv_replies);
+        local->replies = GF_CALLOC (num_stripe, sizeof (struct stripe_replies),
+                                    gf_stripe_mt_stripe_replies);
         if (!local->replies) {
                 op_errno = ENOMEM;
                 goto err;
@@ -3543,7 +3544,11 @@ stripe_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 {
         int32_t         callcnt = 0;
         stripe_local_t *local = NULL;
+	stripe_local_t *mlocal = NULL;
         call_frame_t   *prev = NULL;
+	call_frame_t   *mframe = NULL;
+	struct stripe_replies *reply = NULL;
+	int32_t		i = 0;
 
         if (!this || !frame || !frame->local || !cookie) {
                 gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
@@ -3552,48 +3557,75 @@ stripe_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         prev  = cookie;
         local = frame->local;
+	mframe = local->orig_frame;
+	mlocal = mframe->local;
 
         LOCK(&frame->lock);
         {
-                callcnt = ++local->call_count;
+                callcnt = ++mlocal->call_count;
+
+		mlocal->replies[local->node_index].op_ret = op_ret;
+		mlocal->replies[local->node_index].op_errno = op_errno;
 
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s returned error %s",
-                                prev->this->name, strerror (op_errno));
-                        local->op_errno = op_errno;
-                        local->op_ret = -1;
-                }
                 if (op_ret >= 0) {
-                        local->op_ret += op_ret;
-                        local->post_buf = *postbuf;
-                        local->pre_buf = *prebuf;
+                        mlocal->post_buf = *postbuf;
+                        mlocal->pre_buf = *prebuf;
 
-			local->prebuf_blocks  += prebuf->ia_blocks;
-			local->postbuf_blocks += postbuf->ia_blocks;
+			mlocal->prebuf_blocks  += prebuf->ia_blocks;
+			mlocal->postbuf_blocks += postbuf->ia_blocks;
 
-			correct_file_size(prebuf, local->fctx, prev);
-			correct_file_size(postbuf, local->fctx, prev);
+			correct_file_size(prebuf, mlocal->fctx, prev);
+			correct_file_size(postbuf, mlocal->fctx, prev);
 
-			if (local->prebuf_size < prebuf->ia_size)
-				local->prebuf_size = prebuf->ia_size;
-			if (local->postbuf_size < postbuf->ia_size)
-				local->postbuf_size = postbuf->ia_size;
+			if (mlocal->prebuf_size < prebuf->ia_size)
+				mlocal->prebuf_size = prebuf->ia_size;
+			if (mlocal->postbuf_size < postbuf->ia_size)
+				mlocal->postbuf_size = postbuf->ia_size;
                 }
         }
         UNLOCK (&frame->lock);
 
-        if ((callcnt == local->wind_count) && local->unwind) {
-		local->pre_buf.ia_size = local->prebuf_size;
-		local->pre_buf.ia_blocks = local->prebuf_blocks;
-		local->post_buf.ia_size = local->postbuf_size;
-		local->post_buf.ia_blocks = local->postbuf_blocks;
+        if ((callcnt == mlocal->wind_count) && mlocal->unwind) {
+		mlocal->pre_buf.ia_size = mlocal->prebuf_size;
+		mlocal->pre_buf.ia_blocks = mlocal->prebuf_blocks;
+		mlocal->post_buf.ia_size = mlocal->postbuf_size;
+		mlocal->post_buf.ia_blocks = mlocal->postbuf_blocks;
+
+		/*
+		 * Only return the number of consecutively written bytes up until
+		 * the first error. Only return an error if it occurs first.
+		 *
+		 * When a short write occurs, the application should retry at the
+		 * appropriate offset, at which point we'll potentially pass back
+		 * the error.
+		 */
+		for (i = 0, reply = mlocal->replies; i < mlocal->wind_count;
+			i++, reply++) {
+			if (reply->op_ret == -1) {
+				gf_log(this->name, GF_LOG_DEBUG, "reply %d "
+					"returned error %s", i,
+					strerror(reply->op_errno));
+				if (!mlocal->op_ret) {
+					mlocal->op_ret = -1;
+					mlocal->op_errno = reply->op_errno;
+				}
+				break;
+			}
 
-                STRIPE_STACK_UNWIND (writev, frame, local->op_ret,
-                                     local->op_errno, &local->pre_buf,
-                                     &local->post_buf, NULL);
+			mlocal->op_ret += reply->op_ret;
+
+			if (reply->op_ret < reply->requested_size)
+				break;
+		}
+
+		GF_FREE(mlocal->replies);
+
+                STRIPE_STACK_UNWIND (writev, mframe, mlocal->op_ret,
+                                     mlocal->op_errno, &mlocal->pre_buf,
+                                     &mlocal->post_buf, NULL);
         }
 out:
+	STRIPE_STACK_DESTROY(frame);
         return 0;
 }
 
@@ -3615,6 +3647,11 @@ stripe_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
         uint64_t          stripe_size = 0;
         uint64_t          tmp_fctx = 0;
 	off_t		  dest_offset = 0;
+	off_t		  rounded_start = 0;
+	off_t		  rounded_end = 0;
+	int32_t		  total_chunks = 0;
+	call_frame_t	  *wframe = NULL;
+	stripe_local_t	  *wlocal = NULL;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
@@ -3653,7 +3690,27 @@ stripe_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
                 goto err;
         }
 
+	rounded_start = floor(offset, stripe_size);
+	rounded_end = roof(offset + total_size, stripe_size);
+	total_chunks = (rounded_end - rounded_start) / stripe_size;
+	local->replies = GF_CALLOC(total_chunks, sizeof(struct stripe_replies),
+				gf_stripe_mt_stripe_replies);
+	if (!local->replies) {
+		op_errno = ENOMEM;
+		goto err;
+	}
+
+	total_chunks = 0;
         while (1) {
+		wframe = copy_frame(frame);
+		wlocal = mem_get0(this->local_pool);
+		if (!wlocal) {
+			op_errno = ENOMEM;
+			goto err;
+		}
+		wlocal->orig_frame = frame;
+		wframe->local = wlocal;
+
                 /* Send striped chunk of the vector to child
                    nodes appropriately. */
                 idx = (((offset + offset_offset) /
@@ -3681,31 +3738,539 @@ stripe_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
                 if (remaining_size == 0)
                         local->unwind = 1;
 
+		/*
+		 * Store off the request index (with respect to the chunk of the
+		 * initial offset) and the size of the request. This is required
+		 * in the callback to calculate an appropriate return value in
+		 * the event of a write failure in one or more requests.
+		 */
+		wlocal->node_index = total_chunks;
+		local->replies[total_chunks].requested_size = fill_size;
+
+		dest_offset = offset + offset_offset;
 		if (fctx->stripe_coalesce)
-			dest_offset = coalesced_offset(offset + offset_offset,
-				local->stripe_size, fctx->stripe_count);
-		else
-			dest_offset = offset + offset_offset;
+			dest_offset = coalesced_offset(dest_offset,
+					local->stripe_size, fctx->stripe_count);
 
-                STACK_WIND (frame, stripe_writev_cbk, fctx->xl_array[idx],
+                STACK_WIND (wframe, stripe_writev_cbk, fctx->xl_array[idx],
                             fctx->xl_array[idx]->fops->writev, fd, tmp_vec,
                             tmp_count, dest_offset, flags, iobref,
                             xdata);
 
                 GF_FREE (tmp_vec);
                 offset_offset += fill_size;
+		total_chunks++;
                 if (remaining_size == 0)
                         break;
         }
 
         return 0;
 err:
+	if (wframe)
+		STRIPE_STACK_DESTROY(wframe);
+
         STRIPE_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
 
 int32_t
+stripe_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
+{
+        int32_t         callcnt = 0;
+        stripe_local_t *local = NULL;
+	stripe_local_t *mlocal = NULL;
+        call_frame_t   *prev = NULL;
+	call_frame_t   *mframe = NULL;
+
+        if (!this || !frame || !frame->local || !cookie) {
+                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
+                goto out;
+        }
+
+        prev  = cookie;
+        local = frame->local;
+	mframe = local->orig_frame;
+	mlocal = mframe->local;
+
+        LOCK(&frame->lock);
+        {
+                callcnt = ++mlocal->call_count;
+
+                if (op_ret == 0) {
+                        mlocal->post_buf = *postbuf;
+                        mlocal->pre_buf = *prebuf;
+
+			mlocal->prebuf_blocks  += prebuf->ia_blocks;
+			mlocal->postbuf_blocks += postbuf->ia_blocks;
+
+			correct_file_size(prebuf, mlocal->fctx, prev);
+			correct_file_size(postbuf, mlocal->fctx, prev);
+
+			if (mlocal->prebuf_size < prebuf->ia_size)
+				mlocal->prebuf_size = prebuf->ia_size;
+			if (mlocal->postbuf_size < postbuf->ia_size)
+				mlocal->postbuf_size = postbuf->ia_size;
+                }
+
+		/* return the first failure */
+		if (mlocal->op_ret == 0) {
+			mlocal->op_ret = op_ret;
+			mlocal->op_errno = op_errno;
+		}
+        }
+        UNLOCK (&frame->lock);
+
+        if ((callcnt == mlocal->wind_count) && mlocal->unwind) {
+		mlocal->pre_buf.ia_size = mlocal->prebuf_size;
+		mlocal->pre_buf.ia_blocks = mlocal->prebuf_blocks;
+		mlocal->post_buf.ia_size = mlocal->postbuf_size;
+		mlocal->post_buf.ia_blocks = mlocal->postbuf_blocks;
+
+                STRIPE_STACK_UNWIND (fallocate, mframe, mlocal->op_ret,
+                                     mlocal->op_errno, &mlocal->pre_buf,
+                                     &mlocal->post_buf, NULL);
+        }
+out:
+	STRIPE_STACK_DESTROY(frame);
+        return 0;
+}
+
+int32_t
+stripe_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+		 off_t offset, size_t len, dict_t *xdata)
+{
+        stripe_local_t   *local = NULL;
+        stripe_fd_ctx_t  *fctx = NULL;
+        int32_t           op_errno = 1;
+        int32_t           idx = 0;
+        int32_t           offset_offset = 0;
+        int32_t           remaining_size = 0;
+        off_t             fill_size = 0;
+        uint64_t          stripe_size = 0;
+        uint64_t          tmp_fctx = 0;
+	off_t		  dest_offset = 0;
+	call_frame_t	  *fframe = NULL;
+	stripe_local_t	  *flocal = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+        VALIDATE_OR_GOTO (fd->inode, err);
+
+        inode_ctx_get (fd->inode, this, &tmp_fctx);
+        if (!tmp_fctx) {
+                op_errno = EINVAL;
+                goto err;
+        }
+        fctx = (stripe_fd_ctx_t *)(long)tmp_fctx;
+        stripe_size = fctx->stripe_size;
+
+        STRIPE_VALIDATE_FCTX (fctx, err);
+
+        remaining_size = len;
+
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+        frame->local = local;
+        local->stripe_size = stripe_size;
+	local->fctx = fctx;
+
+        if (!stripe_size) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Wrong stripe size for the file");
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        while (1) {
+		fframe = copy_frame(frame);
+		flocal = mem_get0(this->local_pool);
+		if (!flocal) {
+			op_errno = ENOMEM;
+			goto err;
+		}
+		flocal->orig_frame = frame;
+		fframe->local = flocal;
+
+		/* send fallocate request to the associated child node */
+                idx = (((offset + offset_offset) /
+                        local->stripe_size) % fctx->stripe_count);
+
+                fill_size = (local->stripe_size -
+                             ((offset + offset_offset) % local->stripe_size));
+                if (fill_size > remaining_size)
+                        fill_size = remaining_size;
+
+                remaining_size -= fill_size;
+
+                local->wind_count++;
+                if (remaining_size == 0)
+                        local->unwind = 1;
+
+		dest_offset = offset + offset_offset;
+		if (fctx->stripe_coalesce)
+			dest_offset = coalesced_offset(dest_offset,
+					local->stripe_size, fctx->stripe_count);
+
+		/*
+		 * TODO: Create a separate handler for coalesce mode that sends a
+		 * single fallocate per-child (since the ranges are linear).
+		 */
+		STACK_WIND(fframe, stripe_fallocate_cbk, fctx->xl_array[idx],
+			   fctx->xl_array[idx]->fops->fallocate, fd, mode,
+			   dest_offset, fill_size, xdata);
+
+                offset_offset += fill_size;
+                if (remaining_size == 0)
+                        break;
+        }
+
+        return 0;
+err:
+	if (fframe)
+		STRIPE_STACK_DESTROY(fframe);
+
+        STRIPE_STACK_UNWIND (fallocate, frame, -1, op_errno, NULL, NULL, NULL);
+        return 0;
+}
+
+
+int32_t
+stripe_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                   struct iatt *postbuf, dict_t *xdata)
+{
+        int32_t         callcnt = 0;
+        stripe_local_t *local = NULL;
+	stripe_local_t *mlocal = NULL;
+        call_frame_t   *prev = NULL;
+	call_frame_t   *mframe = NULL;
+
+        if (!this || !frame || !frame->local || !cookie) {
+                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
+                goto out;
+        }
+
+        prev  = cookie;
+        local = frame->local;
+	mframe = local->orig_frame;
+	mlocal = mframe->local;
+
+        LOCK(&frame->lock);
+        {
+                callcnt = ++mlocal->call_count;
+
+                if (op_ret == 0) {
+                        mlocal->post_buf = *postbuf;
+                        mlocal->pre_buf = *prebuf;
+
+			mlocal->prebuf_blocks  += prebuf->ia_blocks;
+			mlocal->postbuf_blocks += postbuf->ia_blocks;
+
+			correct_file_size(prebuf, mlocal->fctx, prev);
+			correct_file_size(postbuf, mlocal->fctx, prev);
+
+			if (mlocal->prebuf_size < prebuf->ia_size)
+				mlocal->prebuf_size = prebuf->ia_size;
+			if (mlocal->postbuf_size < postbuf->ia_size)
+				mlocal->postbuf_size = postbuf->ia_size;
+                }
+
+		/* return the first failure */
+		if (mlocal->op_ret == 0) {
+			mlocal->op_ret = op_ret;
+			mlocal->op_errno = op_errno;
+		}
+        }
+        UNLOCK (&frame->lock);
+
+        if ((callcnt == mlocal->wind_count) && mlocal->unwind) {
+		mlocal->pre_buf.ia_size = mlocal->prebuf_size;
+		mlocal->pre_buf.ia_blocks = mlocal->prebuf_blocks;
+		mlocal->post_buf.ia_size = mlocal->postbuf_size;
+		mlocal->post_buf.ia_blocks = mlocal->postbuf_blocks;
+
+                STRIPE_STACK_UNWIND (discard, mframe, mlocal->op_ret,
+                                     mlocal->op_errno, &mlocal->pre_buf,
+                                     &mlocal->post_buf, NULL);
+        }
+out:
+	STRIPE_STACK_DESTROY(frame);
+        return 0;
+}
+
+int32_t
+stripe_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+	       size_t len, dict_t *xdata)
+{
+        stripe_local_t   *local = NULL;
+        stripe_fd_ctx_t  *fctx = NULL;
+        int32_t           op_errno = 1;
+        int32_t           idx = 0;
+        int32_t           offset_offset = 0;
+        int32_t           remaining_size = 0;
+        off_t             fill_size = 0;
+        uint64_t          stripe_size = 0;
+        uint64_t          tmp_fctx = 0;
+	off_t		  dest_offset = 0;
+	call_frame_t	  *fframe = NULL;
+	stripe_local_t	  *flocal = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+        VALIDATE_OR_GOTO (fd->inode, err);
+
+        inode_ctx_get (fd->inode, this, &tmp_fctx);
+        if (!tmp_fctx) {
+                op_errno = EINVAL;
+                goto err;
+        }
+        fctx = (stripe_fd_ctx_t *)(long)tmp_fctx;
+        stripe_size = fctx->stripe_size;
+
+        STRIPE_VALIDATE_FCTX (fctx, err);
+
+        remaining_size = len;
+
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+        frame->local = local;
+        local->stripe_size = stripe_size;
+	local->fctx = fctx;
+
+        if (!stripe_size) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Wrong stripe size for the file");
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        while (1) {
+		fframe = copy_frame(frame);
+		flocal = mem_get0(this->local_pool);
+		if (!flocal) {
+			op_errno = ENOMEM;
+			goto err;
+		}
+		flocal->orig_frame = frame;
+		fframe->local = flocal;
+
+		/* send discard request to the associated child node */
+                idx = (((offset + offset_offset) /
+                        local->stripe_size) % fctx->stripe_count);
+
+                fill_size = (local->stripe_size -
+                             ((offset + offset_offset) % local->stripe_size));
+                if (fill_size > remaining_size)
+                        fill_size = remaining_size;
+
+                remaining_size -= fill_size;
+
+                local->wind_count++;
+                if (remaining_size == 0)
+                        local->unwind = 1;
+
+		dest_offset = offset + offset_offset;
+		if (fctx->stripe_coalesce)
+			dest_offset = coalesced_offset(dest_offset,
+					local->stripe_size, fctx->stripe_count);
+
+		/*
+		 * TODO: Create a separate handler for coalesce mode that sends a
+		 * single discard per-child (since the ranges are linear).
+		 */
+		STACK_WIND(fframe, stripe_discard_cbk, fctx->xl_array[idx],
+			   fctx->xl_array[idx]->fops->discard, fd, dest_offset,
+			   fill_size, xdata);
+
+                offset_offset += fill_size;
+                if (remaining_size == 0)
+                        break;
+        }
+
+        return 0;
+err:
+	if (fframe)
+		STRIPE_STACK_DESTROY(fframe);
+
+        STRIPE_STACK_UNWIND (discard, frame, -1, op_errno, NULL, NULL, NULL);
+        return 0;
+}
+
+int32_t
+stripe_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                   struct iatt *postbuf, dict_t *xdata)
+{
+        int32_t         callcnt    = 0;
+        stripe_local_t *local      = NULL;
+        stripe_local_t *mlocal     = NULL;
+        call_frame_t   *prev       = NULL;
+        call_frame_t   *mframe     = NULL;
+
+        GF_ASSERT (frame);
+
+        if (!this || !frame->local || !cookie) {
+                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
+                goto out;
+        }
+
+        prev  = cookie;
+        local = frame->local;
+        mframe = local->orig_frame;
+        mlocal = mframe->local;
+
+        LOCK(&frame->lock);
+        {
+                callcnt = ++mlocal->call_count;
+
+                if (op_ret == 0) {
+                        mlocal->post_buf = *postbuf;
+                        mlocal->pre_buf = *prebuf;
+
+                        mlocal->prebuf_blocks  += prebuf->ia_blocks;
+                        mlocal->postbuf_blocks += postbuf->ia_blocks;
+
+                        correct_file_size(prebuf, mlocal->fctx, prev);
+                        correct_file_size(postbuf, mlocal->fctx, prev);
+
+                        if (mlocal->prebuf_size < prebuf->ia_size)
+                                mlocal->prebuf_size = prebuf->ia_size;
+                        if (mlocal->postbuf_size < postbuf->ia_size)
+                                mlocal->postbuf_size = postbuf->ia_size;
+                }
+
+                /* return the first failure */
+                if (mlocal->op_ret == 0) {
+                        mlocal->op_ret = op_ret;
+                        mlocal->op_errno = op_errno;
+                }
+        }
+        UNLOCK (&frame->lock);
+
+        if ((callcnt == mlocal->wind_count) && mlocal->unwind) {
+                mlocal->pre_buf.ia_size = mlocal->prebuf_size;
+                mlocal->pre_buf.ia_blocks = mlocal->prebuf_blocks;
+                mlocal->post_buf.ia_size = mlocal->postbuf_size;
+                mlocal->post_buf.ia_blocks = mlocal->postbuf_blocks;
+
+                STRIPE_STACK_UNWIND (zerofill, mframe, mlocal->op_ret,
+                                     mlocal->op_errno, &mlocal->pre_buf,
+                                     &mlocal->post_buf, NULL);
+        }
+out:
+        STRIPE_STACK_DESTROY(frame);
+        return 0;
+}
+
+int32_t
+stripe_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+               off_t len, dict_t *xdata)
+{
+        stripe_local_t   *local            = NULL;
+        stripe_fd_ctx_t  *fctx             = NULL;
+        int32_t           op_errno         = 1;
+        int32_t           idx              = 0;
+        int32_t           offset_offset    = 0;
+        int32_t           remaining_size   = 0;
+        off_t             fill_size        = 0;
+        uint64_t          stripe_size      = 0;
+        uint64_t          tmp_fctx         = 0;
+        off_t             dest_offset      = 0;
+        call_frame_t     *fframe           = NULL;
+        stripe_local_t   *flocal           = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+        VALIDATE_OR_GOTO (fd->inode, err);
+
+        inode_ctx_get (fd->inode, this, &tmp_fctx);
+        if (!tmp_fctx) {
+                op_errno = EINVAL;
+                goto err;
+        }
+        fctx = (stripe_fd_ctx_t *)(long)tmp_fctx;
+        stripe_size = fctx->stripe_size;
+
+        STRIPE_VALIDATE_FCTX (fctx, err);
+
+        remaining_size = len;
+
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+        frame->local = local;
+        local->stripe_size = stripe_size;
+        local->fctx = fctx;
+
+        if (!stripe_size) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Wrong stripe size for the file");
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        while (1) {
+                fframe = copy_frame(frame);
+                flocal = mem_get0(this->local_pool);
+                if (!flocal) {
+                        op_errno = ENOMEM;
+                        goto err;
+                }
+                flocal->orig_frame = frame;
+                fframe->local = flocal;
+
+                idx = (((offset + offset_offset) /
+                        local->stripe_size) % fctx->stripe_count);
+
+                fill_size = (local->stripe_size -
+                             ((offset + offset_offset) % local->stripe_size));
+                if (fill_size > remaining_size)
+                        fill_size = remaining_size;
+
+                remaining_size -= fill_size;
+
+                local->wind_count++;
+                if (remaining_size == 0)
+                        local->unwind = 1;
+
+                dest_offset = offset + offset_offset;
+                if (fctx->stripe_coalesce)
+                        dest_offset = coalesced_offset(dest_offset,
+                                                       local->stripe_size,
+                                                       fctx->stripe_count);
+
+                STACK_WIND(fframe, stripe_zerofill_cbk, fctx->xl_array[idx],
+                           fctx->xl_array[idx]->fops->zerofill, fd,
+                           dest_offset, fill_size, xdata);
+                offset_offset += fill_size;
+                if (remaining_size == 0)
+                        break;
+        }
+
+        return 0;
+err:
+        if (fframe)
+                STRIPE_STACK_DESTROY(fframe);
+
+        STRIPE_STACK_UNWIND (zerofill, frame, -1, op_errno, NULL, NULL, NULL);
+        return 0;
+}
+
+int32_t
 stripe_release (xlator_t *this, fd_t *fd)
 {
 	return 0;
@@ -3741,6 +4306,7 @@ notify (xlator_t *this, int32_t event, void *data, ...)
         stripe_private_t *priv = NULL;
         int               down_client = 0;
         int               i = 0;
+        gf_boolean_t      heard_from_all_children = _gf_false;
 
         if (!this)
                 return 0;
@@ -3752,30 +4318,34 @@ notify (xlator_t *this, int32_t event, void *data, ...)
         switch (event)
         {
         case GF_EVENT_CHILD_UP:
-        case GF_EVENT_CHILD_CONNECTING:
         {
                 /* get an index number to set */
                 for (i = 0; i < priv->child_count; i++) {
                         if (data == priv->xl_array[i])
                                 break;
                 }
-                priv->state[i] = 1;
-                for (i = 0; i < priv->child_count; i++) {
-                        if (!priv->state[i])
-                                down_client++;
+
+                if (priv->child_count == i) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "got GF_EVENT_CHILD_UP bad subvolume %s",
+                                data? ((xlator_t *)data)->name: NULL);
+                        break;
                 }
 
                 LOCK (&priv->lock);
                 {
-                        priv->nodes_down = down_client;
                         if (data == FIRST_CHILD (this))
                                 priv->first_child_down = 0;
-                        if (!priv->nodes_down)
-                                default_notify (this, event, data);
+                        priv->last_event[i] = event;
                 }
                 UNLOCK (&priv->lock);
         }
         break;
+        case GF_EVENT_CHILD_CONNECTING:
+        {
+                // 'CONNECTING' doesn't ensure its CHILD_UP, so do nothing
+                goto out;
+        }
         case GF_EVENT_CHILD_DOWN:
         {
                 /* get an index number to set */
@@ -3783,20 +4353,19 @@ notify (xlator_t *this, int32_t event, void *data, ...)
                         if (data == priv->xl_array[i])
                                 break;
                 }
-                priv->state[i] = 0;
-                for (i = 0; i < priv->child_count; i++) {
-                        if (!priv->state[i])
-                                down_client++;
+
+                if (priv->child_count == i) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "got GF_EVENT_CHILD_DOWN bad subvolume %s",
+                                data? ((xlator_t *)data)->name: NULL);
+                        break;
                 }
 
                 LOCK (&priv->lock);
                 {
-                        priv->nodes_down = down_client;
-
                         if (data == FIRST_CHILD (this))
                                 priv->first_child_down = 1;
-                        if (priv->nodes_down)
-                                default_notify (this, event, data);
+                        priv->last_event[i] = event;
                 }
                 UNLOCK (&priv->lock);
         }
@@ -3806,10 +4375,30 @@ notify (xlator_t *this, int32_t event, void *data, ...)
         {
                 /* */
                 default_notify (this, event, data);
+                goto out;
         }
         break;
         }
 
+        // Consider child as down if it's last_event is not CHILD_UP
+        for (i = 0, down_client = 0; i < priv->child_count; i++)
+                if (priv->last_event[i] != GF_EVENT_CHILD_UP)
+                        down_client++;
+
+        LOCK (&priv->lock);
+        {
+                priv->nodes_down = down_client;
+        }
+        UNLOCK (&priv->lock);
+
+        heard_from_all_children = _gf_true;
+        for (i = 0; i < priv->child_count; i++)
+                if (!priv->last_event[i])
+                        heard_from_all_children = _gf_false;
+
+        if (heard_from_all_children)
+                default_notify (this, event, data);
+out:
         return 0;
 }
 
@@ -3855,16 +4444,47 @@ stripe_setxattr_cbk (call_frame_t *frame, void *cookie,
         return 0;
 }
 
+#ifdef HAVE_BD_XLATOR
+int
+stripe_is_bd (dict_t *this, char *key, data_t *value, void *data)
+{
+        gf_boolean_t *is_bd = data;
+
+        if (data == NULL)
+                return 0;
+
+        if (XATTR_IS_BD (key))
+            *is_bd = _gf_true;
+
+        return 0;
+}
+
+inline gf_boolean_t
+stripe_setxattr_is_bd (dict_t *dict)
+{
+        gf_boolean_t is_bd = _gf_false;
+
+        if (dict == NULL)
+                goto out;
+
+        dict_foreach (dict, stripe_is_bd, &is_bd);
+out:
+        return is_bd;
+}
+#else
+#define stripe_setxattr_is_bd(dict) _gf_false
+#endif
+
 int
 stripe_setxattr (call_frame_t *frame, xlator_t *this,
                  loc_t *loc, dict_t *dict, int flags, dict_t *xdata)
 {
-        data_pair_t      *pair     = NULL;
         int32_t           op_errno = EINVAL;
         xlator_list_t    *trav     = NULL;
         stripe_private_t *priv     = NULL;
         stripe_local_t   *local    = NULL;
         int               i        = 0;
+        gf_boolean_t      is_bd    = _gf_false;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
@@ -3872,7 +4492,7 @@ stripe_setxattr (call_frame_t *frame, xlator_t *this,
         VALIDATE_OR_GOTO (loc->inode, err);
 
         GF_IF_INTERNAL_XATTR_GOTO ("trusted.*stripe*", dict,
-                                   pair, op_errno, err);
+                                   op_errno, err);
 
         priv = this->private;
         trav = this->children;
@@ -3887,11 +4507,15 @@ stripe_setxattr (call_frame_t *frame, xlator_t *this,
         local->wind_count = priv->child_count;
         local->op_ret = local->op_errno = 0;
 
+        is_bd = stripe_setxattr_is_bd (dict);
+
         /**
          * Set xattrs for directories on all subvolumes. Additionally
-         * this power is only given to a special client.
+         * this power is only given to a special client. Bd xlator
+         * also needs xattrs for regular files (ie LVs)
          */
-        if ((frame->root->pid == -1) && IA_ISDIR (loc->inode->ia_type)) {
+        if (((frame->root->pid == GF_CLIENT_PID_GSYNCD) &&
+             IA_ISDIR (loc->inode->ia_type)) || is_bd) {
                 for (i = 0; i < priv->child_count; i++, trav = trav->next) {
                         STACK_WIND (frame, stripe_setxattr_cbk,
                                     trav->xlator, trav->xlator->fops->setxattr,
@@ -3920,27 +4544,138 @@ stripe_fsetxattr_cbk (call_frame_t *frame, void *cookie,
         return 0;
 }
 
+
+int
+stripe_is_special_key (dict_t *this,
+                       char *key,
+                       data_t *value,
+                       void *data)
+{
+        gf_boolean_t *is_special = NULL;
+
+        if (data == NULL) {
+                goto out;
+        }
+
+        is_special = data;
+
+        if (XATTR_IS_LOCKINFO (key) || XATTR_IS_BD (key))
+                *is_special = _gf_true;
+
+out:
+        return 0;
+}
+
+int32_t
+stripe_fsetxattr_everyone_cbk (call_frame_t *frame, void *cookie,
+                               xlator_t *this, int32_t op_ret, int32_t op_errno,
+                               dict_t *xdata)
+{
+        int             call_count = 0;
+        stripe_local_t *local      = NULL;
+
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                call_count = --local->wind_count;
+
+                if (op_ret < 0) {
+                        local->op_ret = op_ret;
+                        local->op_errno = op_errno;
+                }
+        }
+        UNLOCK (&frame->lock);
+
+        if (call_count == 0) {
+                STRIPE_STACK_UNWIND (fsetxattr, frame, local->op_ret,
+                                     local->op_errno, NULL);
+        }
+        return 0;
+}
+
+int
+stripe_fsetxattr_to_everyone (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                              dict_t *dict, int flags, dict_t *xdata)
+{
+        xlator_list_t    *trav  = NULL;
+        stripe_private_t *priv  = NULL;
+        int               ret   = -1;
+        stripe_local_t   *local = NULL;
+
+        priv = this->private;
+
+        local = mem_get0 (this->local_pool);
+        if (local == NULL) {
+                goto out;
+        }
+
+        frame->local = local;
+
+        local->wind_count = priv->child_count;
+
+        trav = this->children;
+
+        while (trav) {
+                STACK_WIND (frame, stripe_fsetxattr_everyone_cbk,
+                            trav->xlator, trav->xlator->fops->fsetxattr,
+                            fd, dict, flags, xdata);
+                trav = trav->next;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+inline gf_boolean_t
+stripe_fsetxattr_is_special (dict_t *dict)
+{
+        gf_boolean_t is_spl = _gf_false;
+
+        if (dict == NULL) {
+                goto out;
+        }
+
+        dict_foreach (dict, stripe_is_special_key, &is_spl);
+
+out:
+        return is_spl;
+}
+
 int
 stripe_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
                   dict_t *dict, int flags, dict_t *xdata)
 {
-        data_pair_t    *trav     = NULL;
-        int32_t         op_ret   = -1;
-        int32_t         op_errno = EINVAL;
+        int32_t      op_ret = -1, ret = -1, op_errno = EINVAL;
+        gf_boolean_t is_spl = _gf_false;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (fd, err);
 
         GF_IF_INTERNAL_XATTR_GOTO ("trusted.*stripe*", dict,
-                                   trav, op_errno, err);
+                                   op_errno, err);
+
+        is_spl = stripe_fsetxattr_is_special (dict);
+        if (is_spl) {
+                ret = stripe_fsetxattr_to_everyone (frame, this, fd, dict,
+                                                    flags, xdata);
+                if (ret < 0) {
+                        op_errno = ENOMEM;
+                        goto err;
+                }
+
+                goto out;
+        }
 
         STACK_WIND (frame, stripe_fsetxattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fsetxattr,
                     fd, dict, flags, xdata);
+out:
         return 0;
- err:
+err:
         STRIPE_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno, NULL);
         return 0;
 }
@@ -4151,7 +4886,7 @@ unlock:
 
                 if (!local_entry)
                         break;
-                if (!IA_ISREG (local_entry->d_stat.ia_type)) {
+                if (!IA_ISREG (local_entry->d_stat.ia_type) || !local_entry->inode) {
                         LOCK (&frame->lock);
                         {
                                 local->wind_count--;
@@ -4420,9 +5155,9 @@ init (xlator_t *this)
         if (!priv->xl_array)
                 goto out;
 
-        priv->state = GF_CALLOC (count, sizeof (int8_t),
-                                 gf_stripe_mt_int8_t);
-        if (!priv->state)
+        priv->last_event = GF_CALLOC (count, sizeof (int),
+                                      gf_stripe_mt_int32_t);
+        if (!priv->last_event)
                 goto out;
 
         priv->child_count = count;
@@ -4491,8 +5226,7 @@ init (xlator_t *this)
 out:
         if (ret) {
                 if (priv) {
-                        if (priv->xl_array)
-                                GF_FREE (priv->xl_array);
+                        GF_FREE (priv->xl_array);
                         GF_FREE (priv);
                 }
         }
@@ -4516,8 +5250,7 @@ fini (xlator_t *this)
         priv = this->private;
         if (priv) {
                 this->private = NULL;
-                if (priv->xl_array)
-                        GF_FREE (priv->xl_array);
+                GF_FREE (priv->xl_array);
 
                 trav = priv->pattern;
                 while (trav) {
@@ -4525,6 +5258,7 @@ fini (xlator_t *this)
                         trav = trav->next;
                         GF_FREE (prev);
                 }
+                GF_FREE (priv->last_event);
                 LOCK_DESTROY (&priv->lock);
                 GF_FREE (priv);
         }
@@ -4562,7 +5296,7 @@ stripe_internal_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         sprintf (size_key, "trusted.%s.stripe-size", this->name);
         sprintf (count_key, "trusted.%s.stripe-count", this->name);
         sprintf (index_key, "trusted.%s.stripe-index", this->name);
-        sprintf (coalesce_key, "trusted.%s.stripe-coalesce", this->name);
+	sprintf (coalesce_key, "trusted.%s.stripe-coalesce", this->name);
 
         dict_del (xattr, size_key);
         dict_del (xattr, count_key);
@@ -4623,8 +5357,8 @@ stripe_vgetxattr_cbk (call_frame_t *frame, void *cookie,
         int32_t              callcnt       = 0;
         int32_t              ret           = -1;
         long                 cky           = 0;
-        char                *xattr_val     = NULL;
-        char                *xattr_serz    = NULL;
+        void                *xattr_val     = NULL;
+        void                *xattr_serz    = NULL;
         stripe_xattr_sort_t *xattr         = NULL;
         dict_t              *stripe_xattr  = NULL;
 
@@ -4655,18 +5389,20 @@ stripe_vgetxattr_cbk (call_frame_t *frame, void *cookie,
                                            gf_stripe_mt_xattr_sort_t);
 
                 if (local->xattr_list) {
-                        ret = dict_get_str (dict, local->xsel, &xattr_val);
-                        if (ret)
-                                goto out;
-
                         xattr = local->xattr_list + (int32_t) cky;
 
-                        xattr_val = gf_strdup (xattr_val);
+                        ret = dict_get_ptr_and_len (dict, local->xsel,
+                                                    &xattr_val,
+                                                    &xattr->xattr_len);
+                        if (xattr->xattr_len == 0)
+                                goto out;
+
                         xattr->pos = cky;
-                        xattr->xattr_value = xattr_val;
-                        xattr->xattr_len = strlen (xattr_val);
+                        xattr->xattr_value = gf_memdup (xattr_val,
+                                                        xattr->xattr_len);
 
-                        local->xattr_total_len += xattr->xattr_len + 1;
+                        if (xattr->xattr_value != NULL)
+                                local->xattr_total_len += xattr->xattr_len + 1;
                 }
         }
  out:
@@ -4683,19 +5419,24 @@ stripe_vgetxattr_cbk (call_frame_t *frame, void *cookie,
                 /* select filler based on ->xsel */
                 if (XATTR_IS_PATHINFO (local->xsel))
                         ret = stripe_fill_pathinfo_xattr (this, local,
+                                                          (char **)&xattr_serz);
+                else if (XATTR_IS_LOCKINFO (local->xsel)) {
+                        ret = stripe_fill_lockinfo_xattr (this, local,
                                                           &xattr_serz);
-                else {
+                } else {
                         gf_log (this->name, GF_LOG_WARNING,
                                 "Unknown xattr in xattr request");
                         goto unwind;
                 }
 
                 if (!ret) {
-                        ret = dict_set_dynstr (stripe_xattr, local->xsel,
-                                               xattr_serz);
+                        ret = dict_set_dynptr (stripe_xattr, local->xsel,
+                                               xattr_serz,
+                                               local->xattr_total_len);
                         if (ret)
                                 gf_log (this->name, GF_LOG_ERROR,
-                                        "Can't set %s key in dict", local->xsel);
+                                        "Can't set %s key in dict",
+                                        local->xsel);
                 }
 
         unwind:
@@ -4704,8 +5445,7 @@ stripe_vgetxattr_cbk (call_frame_t *frame, void *cookie,
 
                 ret = stripe_free_xattr_str (local);
 
-                if (local->xattr_list)
-                        GF_FREE (local->xattr_list);
+                GF_FREE (local->xattr_list);
 
                 if (stripe_xattr)
                         dict_unref (stripe_xattr);
@@ -4747,7 +5487,7 @@ stripe_getxattr (call_frame_t *frame, xlator_t *this,
 
 
         if (name && (strcmp (GF_XATTR_MARKER_KEY, name) == 0)
-            && (-1 == frame->root->pid)) {
+            && (GF_CLIENT_PID_GSYNCD == frame->root->pid)) {
                 local->marker.call_count = priv->child_count;
 
                 sub_volumes = alloca ( priv->child_count *
@@ -4762,7 +5502,8 @@ stripe_getxattr (call_frame_t *frame, xlator_t *this,
                 if (cluster_getmarkerattr (frame, this, loc, name,
                                            local, stripe_getxattr_unwind,
                                            sub_volumes, priv->child_count,
-                                           MARKER_UUID_TYPE, priv->vol_uuid)) {
+                                           MARKER_UUID_TYPE, marker_uuid_default_gauge,
+                                           priv->vol_uuid)) {
                         op_errno = EINVAL;
                         goto err;
                 }
@@ -4784,9 +5525,7 @@ stripe_getxattr (call_frame_t *frame, xlator_t *this,
                 return 0;
         }
 
-        if (name &&
-            ((strncmp (name, GF_XATTR_PATHINFO_KEY,
-                       strlen (GF_XATTR_PATHINFO_KEY)) == 0))) {
+        if (name && (XATTR_IS_PATHINFO (name))) {
                 if (IA_ISREG (loc->inode->ia_type)) {
                         ret = inode_ctx_get (loc->inode, this,
                                              (uint64_t *) &local->fctx);
@@ -4818,7 +5557,7 @@ stripe_getxattr (call_frame_t *frame, xlator_t *this,
 
         if (name &&(*priv->vol_uuid)) {
                 if ((match_uuid_local (name, priv->vol_uuid) == 0)
-                    && (-1 == frame->root->pid)) {
+                    && (GF_CLIENT_PID_GSYNCD == frame->root->pid)) {
 
                         if (!IA_FILE_OR_DIR (loc->inode->ia_type))
                                 local->marker.call_count = 1;
@@ -4841,6 +5580,7 @@ stripe_getxattr (call_frame_t *frame, xlator_t *this,
                                                    sub_volumes,
                                                    local->marker.call_count,
                                                    MARKER_XTIME_TYPE,
+                                                   marker_xtime_default_gauge,
                                                    priv->vol_uuid)) {
                                 op_errno = EINVAL;
                                 goto err;
@@ -4861,6 +5601,78 @@ err:
         return 0;
 }
 
+inline gf_boolean_t
+stripe_is_special_xattr (const char *name)
+{
+        gf_boolean_t    is_spl = _gf_false;
+
+        if (!name) {
+                goto out;
+        }
+
+        if (!strncmp (name, GF_XATTR_LOCKINFO_KEY,
+                      strlen (GF_XATTR_LOCKINFO_KEY))
+            || XATTR_IS_PATHINFO (name))
+                is_spl = _gf_true;
+out:
+        return is_spl;
+}
+
+int32_t
+stripe_fgetxattr_from_everyone (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                                const char *name, dict_t *xdata)
+{
+        stripe_local_t   *local = NULL;
+        stripe_private_t *priv  = NULL;
+        int32_t           ret   = -1, op_errno = 0;
+        int               i     = 0;
+        xlator_list_t    *trav  = NULL;
+
+        priv = this->private;
+
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        local->op_ret = -1;
+        frame->local = local;
+
+        strncpy (local->xsel, name, strlen (name));
+        local->nallocs = local->wind_count = priv->child_count;
+
+        for (i = 0, trav = this->children; i < priv->child_count; i++,
+                     trav = trav->next) {
+                STACK_WIND_COOKIE (frame, stripe_vgetxattr_cbk,
+                                   (void *) (long) i, trav->xlator,
+                                   trav->xlator->fops->fgetxattr,
+                                   fd, name, xdata);
+        }
+
+        return 0;
+
+err:
+        STACK_UNWIND_STRICT (fgetxattr, frame, -1, op_errno, NULL, NULL);
+        return ret;
+}
+
+int32_t
+stripe_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                  const char *name, dict_t *xdata)
+{
+        if (stripe_is_special_xattr (name)) {
+                stripe_fgetxattr_from_everyone (frame, this, fd, name, xdata);
+                goto out;
+        }
+
+        STACK_WIND (frame, stripe_internal_getxattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fgetxattr, fd, name, xdata);
+
+out:
+        return 0;
+}
+
 
 
 int32_t
@@ -4937,9 +5749,13 @@ struct xlator_fops fops = {
         .setxattr       = stripe_setxattr,
         .fsetxattr      = stripe_fsetxattr,
         .getxattr       = stripe_getxattr,
+        .fgetxattr      = stripe_fgetxattr,
         .removexattr    = stripe_removexattr,
         .fremovexattr   = stripe_fremovexattr,
         .readdirp       = stripe_readdirp,
+	.fallocate	= stripe_fallocate,
+	.discard	= stripe_discard,
+        .zerofill       = stripe_zerofill,
 };
 
 struct xlator_cbks cbks = {
@@ -4965,10 +5781,10 @@ struct volume_options options[] = {
         },
 	{ .key = {"coalesce"},
 	  .type = GF_OPTION_TYPE_BOOL,
-	  .default_value = "false",
-	  .description = "Enable coalesce mode to flatten striped files as "
-			 "stored on the server (i.e., eliminate holes caused "
-			 "by the traditional format)."
+	  .default_value = "true",
+	  .description = "Enable/Disable coalesce mode to flatten striped "
+			 "files as stored on the server (i.e., eliminate holes "
+			 "caused by the traditional format)."
 	},
         { .key  = {NULL} },
 };
diff --git a/xlators/cluster/stripe/src/stripe.h b/xlators/cluster/stripe/src/stripe.h
index 1b9e660c1..5673d18f3 100644
--- a/xlators/cluster/stripe/src/stripe.h
+++ b/xlators/cluster/stripe/src/stripe.h
@@ -73,9 +73,9 @@
        } while (0)
 
 typedef struct stripe_xattr_sort {
-        int32_t  pos;
-        int32_t  xattr_len;
-        char    *xattr_value;
+        int   pos;
+        int   xattr_len;
+        char *xattr_value;
 } stripe_xattr_sort_t;
 
 /**
@@ -98,17 +98,17 @@ struct stripe_private {
         gf_lock_t               lock;
         uint8_t                 nodes_down;
         int8_t                  first_child_down;
+        int                    *last_event;
         int8_t                  child_count;
-        int8_t                 *state; /* Current state of child node */
         gf_boolean_t            xattr_supported;  /* default yes */
 	gf_boolean_t		coalesce;
         char                    vol_uuid[UUID_SIZE + 1];
 };
 
 /**
- * Used to keep info about the replies received from fops->readv calls
+ * Used to keep info about the replies received from readv/writev calls
  */
-struct readv_replies {
+struct stripe_replies {
         struct iovec *vector;
         int32_t       count;    //count of vector
         int32_t       op_ret;   //op_ret of readv
@@ -156,7 +156,7 @@ struct stripe_local {
         blkcnt_t             preparent_blocks;
         blkcnt_t             postparent_blocks;
 
-        struct readv_replies *replies;
+        struct stripe_replies *replies;
         struct statvfs        statvfs_buf;
         dir_entry_t          *entry;
 
@@ -262,6 +262,9 @@ int32_t stripe_xattr_aggregate (char *buffer, stripe_local_t *local,
 off_t coalesced_offset(off_t offset, uint64_t stripe_size, int stripe_count);
 off_t uncoalesced_size(off_t size, uint64_t stripe_size, int stripe_count,
 			int stripe_index);
+int32_t
+stripe_fill_lockinfo_xattr (xlator_t *this, stripe_local_t *local,
+                            void **xattr_serz);
 
 /*
  * Adjust the size attribute for files if coalesce is enabled.
diff --git a/xlators/cluster/unify/src/Makefile.am b/xlators/cluster/unify/src/Makefile.am
deleted file mode 100644
index 2a1fe8372..000000000
--- a/xlators/cluster/unify/src/Makefile.am
+++ /dev/null
@@ -1,16 +0,0 @@
-
-xlator_LTLIBRARIES = unify.la
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/legacy/cluster
-
-unify_la_LDFLAGS = -module -avoidversion
-
-unify_la_SOURCES = unify.c unify-self-heal.c
-unify_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
-
-noinst_HEADERS = unify.h
-
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
-
-CLEANFILES = 
-
diff --git a/xlators/cluster/unify/src/unify-mem-types.h b/xlators/cluster/unify/src/unify-mem-types.h
deleted file mode 100644
index 13c9cc1f7..000000000
--- a/xlators/cluster/unify/src/unify-mem-types.h
+++ /dev/null
@@ -1,41 +0,0 @@
-
-/*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-
-#ifndef __UNIFY_MEM_TYPES_H__
-#define __UNIFY_MEM_TYPES_H__
-
-#include "mem-types.h"
-
-enum gf_unify_mem_types_ {
-        gf_unify_mt_char = gf_common_mt_end + 1,
-        gf_unify_mt_int16_t,
-        gf_unify_mt_xlator_t,
-        gf_unify_mt_unify_private_t,
-        gf_unify_mt_xlator_list_t,
-        gf_unify_mt_dir_entry_t,
-        gf_unify_mt_off_t,
-        gf_unify_mt_int,
-        gf_unify_mt_unify_self_heal_struct,
-        gf_unify_mt_unify_local_t,
-        gf_unify_mt_end
-};
-#endif
-
diff --git a/xlators/cluster/unify/src/unify-self-heal.c b/xlators/cluster/unify/src/unify-self-heal.c
deleted file mode 100644
index f99e4c7c3..000000000
--- a/xlators/cluster/unify/src/unify-self-heal.c
+++ /dev/null
@@ -1,1239 +0,0 @@
-/*
-  Copyright (c) 2007-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-/**
- * unify-self-heal.c : 
- *   This file implements few functions which enables 'unify' translator 
- *  to be consistent in its behaviour when 
- *     > a node fails, 
- *     > a node gets added, 
- *     > a failed node comes back
- *     > a new namespace server is added (ie, an fresh namespace server).
- * 
- *  This functionality of 'unify' will enable glusterfs to support storage
- *  system failure, and maintain consistancy. This works both ways, ie, when
- *  an entry (either file or directory) is found on namespace server, and not
- *  on storage nodes, its created in storage nodes and vica-versa.
- * 
- *  The two fops, where it can be implemented are 'getdents ()' and 'lookup ()'
- *
- */
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "unify.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "stack.h"
-#include "common-utils.h"
-
-int32_t
-unify_sh_getdents_cbk (call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       dir_entry_t *entry,
-		       int32_t count);
-
-int32_t
-unify_sh_ns_getdents_cbk (call_frame_t *frame,
-			  void *cookie,
-			  xlator_t *this,
-			  int32_t op_ret,
-			  int32_t op_errno,
-			  dir_entry_t *entry,
-			  int32_t count);
-
-int32_t
-unify_bgsh_getdents_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno,
-			 dir_entry_t *entry,
-			 int32_t count);
-
-int32_t
-unify_bgsh_ns_getdents_cbk (call_frame_t *frame,
-			    void *cookie,
-			    xlator_t *this,
-			    int32_t op_ret,
-			    int32_t op_errno,
-			    dir_entry_t *entry,
-			    int32_t count);
-
-/**
- * unify_local_wipe - free all the extra allocation of local->* here.
- */
-static void 
-unify_local_wipe (unify_local_t *local)
-{
-	/* Free the strdup'd variables in the local structure */
-	if (local->name) {
-		GF_FREE (local->name);
-	}
-
-	if (local->sh_struct) {
-		if (local->sh_struct->offset_list)
-			GF_FREE (local->sh_struct->offset_list);
-
-		if (local->sh_struct->entry_list)
-			GF_FREE (local->sh_struct->entry_list);
-
-		if (local->sh_struct->count_list)
-			GF_FREE (local->sh_struct->count_list);
-
-		GF_FREE (local->sh_struct);
-	}
-
-	loc_wipe (&local->loc1);
-	loc_wipe (&local->loc2);
-}
-
-int32_t 
-unify_sh_setdents_cbk (call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno)
-{
-	int32_t callcnt = -1;
-	unify_local_t *local = frame->local;
-	inode_t *inode = NULL;
-	dict_t *tmp_dict = NULL;
-	dir_entry_t *prev, *entry, *trav;
-
-	LOCK (&frame->lock);
-	{
-		/* if local->call_count == 0, that means, setdents on 
-		 * storagenodes is still pending.
-		 */
-		if (local->call_count)
-			callcnt = --local->call_count;
-	}
-	UNLOCK (&frame->lock);
-
-	if (callcnt == 0) {
-		if (local->sh_struct->entry_list[0]) {
-			prev = entry = local->sh_struct->entry_list[0];
-			if (!entry)
-				return 0;
-			trav = entry->next;
-			while (trav) {
-				prev->next = trav->next;
-				GF_FREE (trav->name);
-				if (IA_ISLNK (trav->buf.ia_type))
-					GF_FREE (trav->link);
-				GF_FREE (trav);
-				trav = prev->next;
-			}
-			GF_FREE (entry);
-		}
-
-		if (!local->flags) {
-			if (local->sh_struct->count_list[0] >= 
-			    UNIFY_SELF_HEAL_GETDENTS_COUNT) {
-				/* count == size, that means, there are more entries
-				   to read from */
-				//local->call_count = 0;
-				local->sh_struct->offset_list[0] += 
-					UNIFY_SELF_HEAL_GETDENTS_COUNT;
-				STACK_WIND (frame,
-					    unify_sh_ns_getdents_cbk,
-					    NS(this),
-					    NS(this)->fops->getdents,
-					    local->fd,
-					    UNIFY_SELF_HEAL_GETDENTS_COUNT,
-					    local->sh_struct->offset_list[0],
-					    GF_GET_DIR_ONLY);
-			}		
-		} else {
-			inode = local->loc1.inode;
-			fd_unref (local->fd);
-			tmp_dict = local->dict;
-
-			unify_local_wipe (local);
-			
-			STACK_UNWIND (frame, local->op_ret, local->op_errno, 
-				      inode, &local->stbuf, local->dict,
-                                      &local->oldpostparent);
-			if (tmp_dict)
-				dict_unref (tmp_dict);
-		}
-	}
-  
-	return 0;
-}
-
-
-int32_t
-unify_sh_ns_getdents_cbk (call_frame_t *frame,
-			  void *cookie,
-			  xlator_t *this,
-			  int32_t op_ret,
-			  int32_t op_errno,
-			  dir_entry_t *entry,
-			  int32_t count)
-{
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	long index = 0;
-	unsigned long final = 0;
-	dir_entry_t *tmp = GF_CALLOC (1, sizeof (dir_entry_t),
-                                      gf_unify_mt_dir_entry_t);
-	
-	local->sh_struct->entry_list[0] = tmp;
-	local->sh_struct->count_list[0] = count;
-	if (entry) {
-		tmp->next = entry->next;
-		entry->next = NULL;
-	}
-
-	if ((count < UNIFY_SELF_HEAL_GETDENTS_COUNT) || !entry) {
-		final = 1;
-	}
-
-	LOCK (&frame->lock);
-	{
-		/* local->call_count will be '0' till now. make it 1 so, it 
-		   can be UNWIND'ed for the last call. */
-		local->call_count = priv->child_count;
-		if (final)
-			local->flags = 1;
-	}
-	UNLOCK (&frame->lock);
-
-	for (index = 0; index < priv->child_count; index++) 
-	{
-		STACK_WIND_COOKIE (frame,
-				   unify_sh_setdents_cbk, 
-				   (void *)index,
-				   priv->xl_array[index],
-				   priv->xl_array[index]->fops->setdents,
-				   local->fd, GF_SET_DIR_ONLY,
-				   local->sh_struct->entry_list[0], count);
-	}
-
-	return 0;
-}
-
-int32_t 
-unify_sh_ns_setdents_cbk (call_frame_t *frame,
-			  void *cookie,
-			  xlator_t *this,
-			  int32_t op_ret,
-			  int32_t op_errno)
-{
-	int32_t callcnt = -1;
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	long index = (long)cookie;
-	dir_entry_t *prev, *entry, *trav;
-
-	LOCK (&frame->lock);
-	{
-		if (local->sh_struct->entry_list[index]) {
-			prev = entry = local->sh_struct->entry_list[index];
-			trav = entry->next;
-			while (trav) {
-				prev->next = trav->next;
-				GF_FREE (trav->name);
-				if (IA_ISLNK (trav->buf.ia_type))
-					GF_FREE (trav->link);
-				GF_FREE (trav);
-				trav = prev->next;
-			}
-			GF_FREE (entry);
-		}
-	}
-	UNLOCK (&frame->lock);
-
-	if (local->sh_struct->count_list[index] < 
-	    UNIFY_SELF_HEAL_GETDENTS_COUNT) {
-		LOCK (&frame->lock);
-		{
-			callcnt = --local->call_count;
-		}
-		UNLOCK (&frame->lock);
-	} else {
-		/* count == size, that means, there are more entries 
-		   to read from */
-		local->sh_struct->offset_list[index] += 
-			UNIFY_SELF_HEAL_GETDENTS_COUNT;
-		STACK_WIND_COOKIE (frame,
-				   unify_sh_getdents_cbk,
-				   cookie,
-				   priv->xl_array[index],
-				   priv->xl_array[index]->fops->getdents,
-				   local->fd,
-				   UNIFY_SELF_HEAL_GETDENTS_COUNT,
-				   local->sh_struct->offset_list[index],
-				   GF_GET_ALL);
-    
-		gf_log (this->name, GF_LOG_DEBUG, 
-			"readdir on (%s) with offset %"PRId64"", 
-			priv->xl_array[index]->name, 
-			local->sh_struct->offset_list[index]);
-	}
-
-	if (!callcnt) {
-		/* All storage nodes have done unified setdents on NS node.
-		 * Now, do getdents from NS and do setdents on storage nodes.
-		 */
-    
-		/* sh_struct->offset_list is no longer required for
-		   storage nodes now */
-		local->sh_struct->offset_list[0] = 0; /* reset */
-
-		STACK_WIND (frame,
-			    unify_sh_ns_getdents_cbk,
-			    NS(this),
-			    NS(this)->fops->getdents,
-			    local->fd,
-			    UNIFY_SELF_HEAL_GETDENTS_COUNT,
-			    0, /* In this call, do send '0' as offset */
-			    GF_GET_DIR_ONLY);
-	}
-
-	return 0;
-}
-
-
-/**
- * unify_sh_getdents_cbk -
- */
-int32_t
-unify_sh_getdents_cbk (call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       dir_entry_t *entry,
-		       int32_t count)
-{
-	int32_t callcnt = -1;
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	long index = (long)cookie;
-	dir_entry_t *tmp = NULL; 
-
-	if (op_ret >= 0 && count > 0) {
-		/* There is some dentry found, just send the dentry to NS */
-		tmp = GF_CALLOC (1, sizeof (dir_entry_t),
-                                 gf_unify_mt_dir_entry_t);
-		local->sh_struct->entry_list[index] = tmp;
-		local->sh_struct->count_list[index] = count;
-		if (entry) {
-			tmp->next = entry->next;
-			entry->next = NULL;
-		}
-		STACK_WIND_COOKIE (frame,
-				   unify_sh_ns_setdents_cbk,
-				   cookie,
-				   NS(this),
-				   NS(this)->fops->setdents,
-				   local->fd,
-				   GF_SET_IF_NOT_PRESENT,
-				   local->sh_struct->entry_list[index],
-				   count);
-		return 0;
-	}
-  
-	if (count < UNIFY_SELF_HEAL_GETDENTS_COUNT) {
-		LOCK (&frame->lock);
-		{
-			callcnt = --local->call_count;
-		}
-		UNLOCK (&frame->lock);
-	} else {
-		/* count == size, that means, there are more entries 
-		   to read from */
-		local->sh_struct->offset_list[index] += 
-			UNIFY_SELF_HEAL_GETDENTS_COUNT;
-		STACK_WIND_COOKIE (frame,
-				   unify_sh_getdents_cbk,
-				   cookie,
-				   priv->xl_array[index],
-				   priv->xl_array[index]->fops->getdents,
-				   local->fd,
-				   UNIFY_SELF_HEAL_GETDENTS_COUNT,
-				   local->sh_struct->offset_list[index],
-				   GF_GET_ALL);
-    
-		gf_log (this->name, GF_LOG_DEBUG, 
-			"readdir on (%s) with offset %"PRId64"", 
-			priv->xl_array[index]->name, 
-			local->sh_struct->offset_list[index]);
-	}
-
-	if (!callcnt) {
-		/* All storage nodes have done unified setdents on NS node.
-		 * Now, do getdents from NS and do setdents on storage nodes.
-		 */
-    
-		/* sh_struct->offset_list is no longer required for
-		   storage nodes now */
-		local->sh_struct->offset_list[0] = 0; /* reset */
-
-		STACK_WIND (frame,
-			    unify_sh_ns_getdents_cbk,
-			    NS(this),
-			    NS(this)->fops->getdents,
-			    local->fd,
-			    UNIFY_SELF_HEAL_GETDENTS_COUNT,
-			    0, /* In this call, do send '0' as offset */
-			    GF_GET_DIR_ONLY);
-	}
-
-	return 0;
-}
-
-/**
- * unify_sh_opendir_cbk -
- *
- * @cookie: 
- */
-int32_t 
-unify_sh_opendir_cbk (call_frame_t *frame,
-		      void *cookie,
-		      xlator_t *this,
-		      int32_t op_ret,
-		      int32_t op_errno,
-		      fd_t *fd)
-{
-	int32_t callcnt = 0;
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	int16_t index = 0;
-	inode_t *inode = NULL;
-	dict_t *tmp_dict = NULL;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-    
-		if (op_ret >= 0) {
-			local->op_ret = op_ret;
-		} else {
-			gf_log (this->name, GF_LOG_WARNING, "failed");
-			local->failed = 1;
-		}
-	}
-	UNLOCK (&frame->lock);
-  
-	if (!callcnt) {
-		local->call_count = priv->child_count + 1;
-    
-		if (!local->failed) {
-			/* send getdents() namespace after finishing
-			   storage nodes */
-			local->call_count--; 
-      
-			fd_bind (fd);
-
-			if (local->call_count) {
-				/* Used as the offset index. This list keeps
-				 * track of offset sent to each node during
-				 * STACK_WIND.
-				 */
-				local->sh_struct->offset_list = 
-					GF_CALLOC (priv->child_count, 
-					           sizeof (off_t),
-                                                   gf_unify_mt_off_t);
-				ERR_ABORT (local->sh_struct->offset_list);
-	
-				local->sh_struct->entry_list = 
-					GF_CALLOC (priv->child_count, 
-						   sizeof (dir_entry_t *),
-                                                   gf_unify_mt_dir_entry_t);
-				ERR_ABORT (local->sh_struct->entry_list);
-
-				local->sh_struct->count_list = 
-					GF_CALLOC (priv->child_count, 
-						   sizeof (int),
-                                                   gf_unify_mt_int);
-				ERR_ABORT (local->sh_struct->count_list);
-
-				/* Send getdents on all the fds */
-				for (index = 0; 
-				     index < priv->child_count; index++) {
-					STACK_WIND_COOKIE (frame,
-							   unify_sh_getdents_cbk,
-							   (void *)(long)index,
-							   priv->xl_array[index],
-							   priv->xl_array[index]->fops->getdents,
-							   local->fd,
-							   UNIFY_SELF_HEAL_GETDENTS_COUNT,
-							   0, /* In this call, do send '0' as offset */
-							   GF_GET_ALL);
-				}
-
-				/* did stack wind, so no need to unwind here */
-				return 0;
-			} /* (local->call_count) */
-		} /* (!local->failed) */
-
-		/* Opendir failed on one node. */ 
-		inode = local->loc1.inode;
-		fd_unref (local->fd);
-		tmp_dict = local->dict;
-
-		unify_local_wipe (local);
-		/* Only 'self-heal' failed, lookup() was successful. */
-		local->op_ret = 0;
-
-		/* This is lookup_cbk ()'s UNWIND. */
-		STACK_UNWIND (frame, local->op_ret, local->op_errno, inode,
-			      &local->stbuf, local->dict, &local->oldpostparent);
-		if (tmp_dict)
-			dict_unref (tmp_dict);
-	}
-
-	return 0;
-}
-
-/**
- * gf_sh_checksum_cbk - 
- * 
- * @frame: frame used in lookup. get a copy of it, and use that copy.
- * @this: pointer to unify xlator.
- * @inode: pointer to inode, for which the consistency check is required.
- *
- */
-int32_t 
-unify_sh_checksum_cbk (call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       uint8_t *file_checksum,
-		       uint8_t *dir_checksum)
-{
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	int16_t index = 0;
-	int32_t callcnt = 0;
-	inode_t *inode = NULL;
-	dict_t *tmp_dict = NULL;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-		if (op_ret >= 0) {
-			if (NS(this) == (xlator_t *)cookie) {
-				memcpy (local->sh_struct->ns_file_checksum, 
-					file_checksum, NAME_MAX);
-				memcpy (local->sh_struct->ns_dir_checksum, 
-					dir_checksum, NAME_MAX);
-			} else {
-				if (local->entry_count == 0) {
-					/* Initialize the dir_checksum to be 
-					 * used for comparision with other
-					 * storage nodes. Should be done for
-					 * the first successful call *only*. 
-					 */
-                                        /* Using 'entry_count' as a flag */
-					local->entry_count = 1;
-					memcpy (local->sh_struct->dir_checksum,
-						dir_checksum, NAME_MAX);
-				}
-
-				/* Reply from the storage nodes */
-				for (index = 0; 
-				     index < NAME_MAX; index++) {
-					/* Files should be present in
-					   only one node */
-					local->sh_struct->file_checksum[index] ^= file_checksum[index];
-	  
-					/* directory structure should be
-					   same accross */
-					if (local->sh_struct->dir_checksum[index] != dir_checksum[index])
-						local->failed = 1;
-				}
-			}
-		} 
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		for (index = 0; index < NAME_MAX ; index++) {
-			if (local->sh_struct->file_checksum[index] != 
-			    local->sh_struct->ns_file_checksum[index]) {
-				local->failed = 1;
-				break;
-			}
-			if (local->sh_struct->dir_checksum[index] != 
-			    local->sh_struct->ns_dir_checksum[index]) {
-				local->failed = 1;
-				break;
-			}
-		}
-	
-		if (local->failed) {
-			/* Log it, it should be a rare event */
-			gf_log (this->name, GF_LOG_WARNING, 
-				"Self-heal triggered on directory %s", 
-				local->loc1.path);
-
-			/* Any self heal will be done at directory level */
-			local->call_count = 0;
-			local->op_ret = -1;
-			local->failed = 0;
-      
-			local->fd = fd_create (local->loc1.inode, 
-					       frame->root->pid);
-
-			local->call_count = priv->child_count + 1;
-	
-			for (index = 0; 
-			     index < (priv->child_count + 1); index++) {
-				STACK_WIND_COOKIE (frame,
-						   unify_sh_opendir_cbk,
-						   priv->xl_array[index]->name,
-						   priv->xl_array[index],
-						   priv->xl_array[index]->fops->opendir,
-						   &local->loc1,
-						   local->fd);
-			}
-			/* opendir can be done on the directory */
-			return 0;
-		}
-
-		/* no mismatch */
-		inode = local->loc1.inode;
-		tmp_dict = local->dict;
-
-		unify_local_wipe (local);
-
-		/* This is lookup_cbk ()'s UNWIND. */
-		STACK_UNWIND (frame,
-			      local->op_ret,
-			      local->op_errno,
-			      inode,
-			      &local->stbuf,
-			      local->dict, &local->oldpostparent);
-		if (tmp_dict)
-			dict_unref (tmp_dict);
-	}
-
-	return 0;
-}
-
-/* Foreground self-heal part over */
-
-/* Background self-heal part */
-
-int32_t 
-unify_bgsh_setdents_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno)
-{
-	int32_t callcnt = -1;
-	unify_local_t *local = frame->local;
-	dir_entry_t *prev, *entry, *trav;
-
-	LOCK (&frame->lock);
-	{
-		/* if local->call_count == 0, that means, setdents 
-		   on storagenodes is still pending. */
-		if (local->call_count)
-			callcnt = --local->call_count;
-	}
-	UNLOCK (&frame->lock);
-
-
-	if (callcnt == 0) {
-		if (local->sh_struct->entry_list[0]) {
-			prev = entry = local->sh_struct->entry_list[0];
-			trav = entry->next;
-			while (trav) {
-				prev->next = trav->next;
-				GF_FREE (trav->name);
-				if (IA_ISLNK (trav->buf.ia_type))
-					GF_FREE (trav->link);
-				GF_FREE (trav);
-				trav = prev->next;
-			}
-			GF_FREE (entry);
-		}
-
-		if (!local->flags) {
-			if (local->sh_struct->count_list[0] >= 
-			    UNIFY_SELF_HEAL_GETDENTS_COUNT) {
-				/* count == size, that means, there are more
-				   entries to read from */
-				//local->call_count = 0;
-				local->sh_struct->offset_list[0] += 
-					UNIFY_SELF_HEAL_GETDENTS_COUNT;
-				STACK_WIND (frame,
-					    unify_bgsh_ns_getdents_cbk,
-					    NS(this),
-					    NS(this)->fops->getdents,
-					    local->fd,
-					    UNIFY_SELF_HEAL_GETDENTS_COUNT,
-					    local->sh_struct->offset_list[0],
-					    GF_GET_DIR_ONLY);
-			}		
-		} else {
-			fd_unref (local->fd);
-			unify_local_wipe (local);
-			STACK_DESTROY (frame->root);
-		}
-	}
-
-	return 0;
-}
-
-
-int32_t
-unify_bgsh_ns_getdents_cbk (call_frame_t *frame,
-			    void *cookie,
-			    xlator_t *this,
-			    int32_t op_ret,
-			    int32_t op_errno,
-			    dir_entry_t *entry,
-			    int32_t count)
-{
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	long index = 0;
-	unsigned long final = 0;
-	dir_entry_t *tmp = GF_CALLOC (1, sizeof (dir_entry_t),
-                                      gf_unify_mt_dir_entry_t);
-
-	local->sh_struct->entry_list[0] = tmp;
-	local->sh_struct->count_list[0] = count;
-	if (entry) {
-		tmp->next = entry->next;
-		entry->next = NULL;
-	}
-
-	if ((count < UNIFY_SELF_HEAL_GETDENTS_COUNT) || !entry) {
-		final = 1;
-	}
-
-	LOCK (&frame->lock);
-	{
-		/* local->call_count will be '0' till now. make it 1 so, 
-		   it can be UNWIND'ed for the last call. */
-		local->call_count = priv->child_count;
-		if (final)
-			local->flags = 1;
-	}
-	UNLOCK (&frame->lock);
-
-	for (index = 0; index < priv->child_count; index++) 
-	{
-		STACK_WIND_COOKIE (frame,
-				   unify_bgsh_setdents_cbk, 
-				   (void *)index,
-				   priv->xl_array[index],
-				   priv->xl_array[index]->fops->setdents,
-				   local->fd, GF_SET_DIR_ONLY,
-				   local->sh_struct->entry_list[0], count);
-	}
-
-	return 0;
-}
-
-int32_t 
-unify_bgsh_ns_setdents_cbk (call_frame_t *frame,
-			    void *cookie,
-			    xlator_t *this,
-			    int32_t op_ret,
-			    int32_t op_errno)
-{
-	int32_t callcnt = -1;
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	long index = (long)cookie;
-	dir_entry_t *prev, *entry, *trav;
-
-	if (local->sh_struct->entry_list[index]) {
-		prev = entry = local->sh_struct->entry_list[index];
-		if (!entry)
-			return 0;
-		trav = entry->next;
-		while (trav) {
-			prev->next = trav->next;
-			GF_FREE (trav->name);
-			if (IA_ISLNK (trav->buf.ia_type))
-				GF_FREE (trav->link);
-			GF_FREE (trav);
-			trav = prev->next;
-		}
-		GF_FREE (entry);
-	}
-
-	if (local->sh_struct->count_list[index] < 
-	    UNIFY_SELF_HEAL_GETDENTS_COUNT) {
-		LOCK (&frame->lock);
-		{
-			callcnt = --local->call_count;
-		}
-		UNLOCK (&frame->lock);
-	} else {
-		/* count == size, that means, there are more entries 
-		   to read from */
-		local->sh_struct->offset_list[index] += 
-			UNIFY_SELF_HEAL_GETDENTS_COUNT;
-		STACK_WIND_COOKIE (frame,
-				   unify_bgsh_getdents_cbk,
-				   cookie,
-				   priv->xl_array[index],
-				   priv->xl_array[index]->fops->getdents,
-				   local->fd,
-				   UNIFY_SELF_HEAL_GETDENTS_COUNT,
-				   local->sh_struct->offset_list[index],
-				   GF_GET_ALL);
-    
-		gf_log (this->name, GF_LOG_DEBUG, 
-			"readdir on (%s) with offset %"PRId64"", 
-			priv->xl_array[index]->name, 
-			local->sh_struct->offset_list[index]);
-	}
-
-	if (!callcnt) {
-		/* All storage nodes have done unified setdents on NS node.
-		 * Now, do getdents from NS and do setdents on storage nodes.
-		 */
-    
-		/* sh_struct->offset_list is no longer required for
-		   storage nodes now */
-		local->sh_struct->offset_list[0] = 0; /* reset */
-
-		STACK_WIND (frame,
-			    unify_bgsh_ns_getdents_cbk,
-			    NS(this),
-			    NS(this)->fops->getdents,
-			    local->fd,
-			    UNIFY_SELF_HEAL_GETDENTS_COUNT,
-			    0, /* In this call, do send '0' as offset */
-			    GF_GET_DIR_ONLY);
-	}
-
-	return 0;
-}
-
-
-/**
- * unify_bgsh_getdents_cbk -
- */
-int32_t
-unify_bgsh_getdents_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno,
-			 dir_entry_t *entry,
-			 int32_t count)
-{
-	int32_t callcnt = -1;
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	long index = (long)cookie;
-	dir_entry_t *tmp = NULL; 
-
-	if (op_ret >= 0 && count > 0) {
-		/* There is some dentry found, just send the dentry to NS */
-		tmp = GF_CALLOC (1, sizeof (dir_entry_t),
-                                 gf_unify_mt_dir_entry_t);
-		local->sh_struct->entry_list[index] = tmp;
-		local->sh_struct->count_list[index] = count;
-		if (entry) {
-			tmp->next = entry->next;
-			entry->next = NULL;
-		}
-		STACK_WIND_COOKIE (frame,
-				   unify_bgsh_ns_setdents_cbk,
-				   cookie,
-				   NS(this),
-				   NS(this)->fops->setdents,
-				   local->fd,
-				   GF_SET_IF_NOT_PRESENT,
-				   local->sh_struct->entry_list[index],
-				   count);
-		return 0;
-	}
-  
-	if (count < UNIFY_SELF_HEAL_GETDENTS_COUNT) {
-		LOCK (&frame->lock);
-		{
-			callcnt = --local->call_count;
-		}
-		UNLOCK (&frame->lock);
-	} else {
-		/* count == size, that means, there are more entries to read from */
-		local->sh_struct->offset_list[index] += 
-			UNIFY_SELF_HEAL_GETDENTS_COUNT;
-
-		STACK_WIND_COOKIE (frame,
-				   unify_bgsh_getdents_cbk,
-				   cookie,
-				   priv->xl_array[index],
-				   priv->xl_array[index]->fops->getdents,
-				   local->fd,
-				   UNIFY_SELF_HEAL_GETDENTS_COUNT,
-				   local->sh_struct->offset_list[index],
-				   GF_GET_ALL);
-    
-		gf_log (this->name, GF_LOG_DEBUG, 
-			"readdir on (%s) with offset %"PRId64"", 
-			priv->xl_array[index]->name, 
-			local->sh_struct->offset_list[index]);
-	}
-
-	if (!callcnt) {
-		/* All storage nodes have done unified setdents on NS node.
-		 * Now, do getdents from NS and do setdents on storage nodes.
-		 */
-    
-		/* sh_struct->offset_list is no longer required for 
-		   storage nodes now */
-		local->sh_struct->offset_list[0] = 0; /* reset */
-
-		STACK_WIND (frame,
-			    unify_bgsh_ns_getdents_cbk,
-			    NS(this),
-			    NS(this)->fops->getdents,
-			    local->fd,
-			    UNIFY_SELF_HEAL_GETDENTS_COUNT,
-			    0, /* In this call, do send '0' as offset */
-			    GF_GET_DIR_ONLY);
-	}
-
-	return 0;
-}
-
-/**
- * unify_bgsh_opendir_cbk -
- *
- * @cookie: 
- */
-int32_t 
-unify_bgsh_opendir_cbk (call_frame_t *frame,
-			void *cookie,
-			xlator_t *this,
-			int32_t op_ret,
-			int32_t op_errno,
-			fd_t *fd)
-{
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	int32_t callcnt = 0;
-	int16_t index = 0;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-    
-		if (op_ret >= 0) {
-			local->op_ret = op_ret;
-		} else {
-			local->failed = 1;
-		}
-	}
-	UNLOCK (&frame->lock);
-  
-	if (!callcnt) {
-		local->call_count = priv->child_count + 1;
-    
-		if (!local->failed) {
-			/* send getdents() namespace after finishing 
-			   storage nodes */
-			local->call_count--; 
-			callcnt = local->call_count;
-      
-			fd_bind (fd);
-
-			if (local->call_count) {
-				/* Used as the offset index. This list keeps 
-				   track of offset sent to each node during 
-				   STACK_WIND. */
-				local->sh_struct->offset_list = 
-					GF_CALLOC (priv->child_count, 
-					           sizeof (off_t),
-                                                   gf_unify_mt_off_t);
-				ERR_ABORT (local->sh_struct->offset_list);
-	
-				local->sh_struct->entry_list = 
-					GF_CALLOC (priv->child_count, 
-						   sizeof (dir_entry_t *),
-                                                   gf_unify_mt_dir_entry_t);
-				ERR_ABORT (local->sh_struct->entry_list);
-
-				local->sh_struct->count_list = 
-					GF_CALLOC (priv->child_count, 
-						   sizeof (int),
-                                                   gf_unify_mt_int);
-				ERR_ABORT (local->sh_struct->count_list);
-
-				/* Send getdents on all the fds */
-				for (index = 0; 
-				     index < priv->child_count; index++) {
-					STACK_WIND_COOKIE (frame,
-							   unify_bgsh_getdents_cbk,
-							   (void *)(long)index,
-							   priv->xl_array[index],
-							   priv->xl_array[index]->fops->getdents,
-							   local->fd,
-							   UNIFY_SELF_HEAL_GETDENTS_COUNT,
-							   0, /* In this call, do send '0' as offset */
-							   GF_GET_ALL);
-				}
-				/* did a stack wind, so no need to unwind here */
-				return 0;
-			} /* (local->call_count) */
-		} /* (!local->failed) */
-
-		/* Opendir failed on one node. 	 */
-		fd_unref (local->fd);
-		
-		unify_local_wipe (local);
-		STACK_DESTROY (frame->root);
-	}
-
-	return 0;
-}
-
-/**
- * gf_bgsh_checksum_cbk - 
- * 
- * @frame: frame used in lookup. get a copy of it, and use that copy.
- * @this: pointer to unify xlator.
- * @inode: pointer to inode, for which the consistency check is required.
- *
- */
-int32_t 
-unify_bgsh_checksum_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno,
-			 uint8_t *file_checksum,
-			 uint8_t *dir_checksum)
-{
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	int16_t index = 0;
-	int32_t callcnt = 0;
-  
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-		if (op_ret >= 0) {
-			if (NS(this) == (xlator_t *)cookie) {
-				memcpy (local->sh_struct->ns_file_checksum, 
-					file_checksum, NAME_MAX);
-				memcpy (local->sh_struct->ns_dir_checksum, 
-					dir_checksum, NAME_MAX);
-			} else {
-				if (local->entry_count == 0) {
-					/* Initialize the dir_checksum to be 
-					 * used for comparision with other 
-					 * storage nodes. Should be done for
-					 * the first successful call *only*. 
-					 */
-					/* Using 'entry_count' as a flag */
-					local->entry_count = 1; 
-					memcpy (local->sh_struct->dir_checksum,
-						dir_checksum, NAME_MAX);
-				}
-
-				/* Reply from the storage nodes */
-				for (index = 0; 
-				     index < NAME_MAX; index++) {
-					/* Files should be present in only 
-					   one node */
-					local->sh_struct->file_checksum[index] ^= file_checksum[index];
-	  
-					/* directory structure should be same 
-					   accross */
-					if (local->sh_struct->dir_checksum[index] != dir_checksum[index])
-						local->failed = 1;
-				}
-			}
-		} 
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		for (index = 0; index < NAME_MAX ; index++) {
-			if (local->sh_struct->file_checksum[index] != 
-			    local->sh_struct->ns_file_checksum[index]) {
-				local->failed = 1;
-				break;
-			}
-			if (local->sh_struct->dir_checksum[index] != 
-			    local->sh_struct->ns_dir_checksum[index]) {
-				local->failed = 1;
-				break;
-			}
-		}
-	
-		if (local->failed) {
-			/* Log it, it should be a rare event */
-			gf_log (this->name, GF_LOG_WARNING, 
-				"Self-heal triggered on directory %s", 
-				local->loc1.path);
-
-			/* Any self heal will be done at the directory level */
-			local->op_ret = -1;
-			local->failed = 0;
-      
-			local->fd = fd_create (local->loc1.inode, 
-					       frame->root->pid);
-			local->call_count = priv->child_count + 1;
-	
-			for (index = 0; 
-			     index < (priv->child_count + 1); index++) {
-				STACK_WIND_COOKIE (frame,
-						   unify_bgsh_opendir_cbk,
-						   priv->xl_array[index]->name,
-						   priv->xl_array[index],
-						   priv->xl_array[index]->fops->opendir,
-						   &local->loc1,
-						   local->fd);
-			}
-      
-			/* opendir can be done on the directory */
-			return 0;
-		}
-
-		/* no mismatch */
-		unify_local_wipe (local);
-		STACK_DESTROY (frame->root);
-	}
-
-	return 0;
-}
-
-/* Background self-heal part over */
-
-
-
-
-/**
- * zr_unify_self_heal - 
- * 
- * @frame: frame used in lookup. get a copy of it, and use that copy.
- * @this: pointer to unify xlator.
- * @inode: pointer to inode, for which the consistency check is required.
- *
- */
-int32_t 
-zr_unify_self_heal (call_frame_t *frame,
-		    xlator_t *this,
-		    unify_local_t *local)
-{
-	unify_private_t *priv = this->private;
-	call_frame_t *bg_frame = NULL;
-	unify_local_t *bg_local = NULL;
-	inode_t *tmp_inode = NULL;
-	dict_t *tmp_dict = NULL;
-	int16_t index = 0;
-  
-	if (local->inode_generation < priv->inode_generation) {
-		/* Any self heal will be done at the directory level */
-		/* Update the inode's generation to the current generation
-		   value. */
-		local->inode_generation = priv->inode_generation;
-		inode_ctx_put (local->loc1.inode, this, 
-			  (uint64_t)(long)local->inode_generation);
-
-		if (priv->self_heal == ZR_UNIFY_FG_SELF_HEAL) {
-			local->op_ret = 0;
-			local->failed = 0;
-			local->call_count = priv->child_count + 1;
-			local->sh_struct = 
-	         	     GF_CALLOC (1, sizeof (struct unify_self_heal_struct),
-                                        gf_unify_mt_unify_self_heal_struct);
-      
-			/* +1 is for NS */
-			for (index = 0; 
-			     index < (priv->child_count + 1); index++) {
-				STACK_WIND_COOKIE (frame,
-						   unify_sh_checksum_cbk,
-						   priv->xl_array[index],
-						   priv->xl_array[index],
-						   priv->xl_array[index]->fops->checksum,
-						   &local->loc1,
-						   0);
-			}
-
-			/* Self-heal in foreground, hence no need 
-			   to UNWIND here */
-			return 0;
-		}
-
-		/* Self Heal done in background */
-		bg_frame = copy_frame (frame);
-		INIT_LOCAL (bg_frame, bg_local);
-		loc_copy (&bg_local->loc1, &local->loc1);
-		bg_local->op_ret = 0;
-		bg_local->failed = 0;
-		bg_local->call_count = priv->child_count + 1;
-		bg_local->sh_struct = 
-			GF_CALLOC (1, sizeof (struct unify_self_heal_struct),
-                                   gf_unify_mt_unify_self_heal_struct);
-    
-		/* +1 is for NS */
-		for (index = 0; index < (priv->child_count + 1); index++) {
-			STACK_WIND_COOKIE (bg_frame,
-					   unify_bgsh_checksum_cbk,
-					   priv->xl_array[index],
-					   priv->xl_array[index],
-					   priv->xl_array[index]->fops->checksum,
-					   &bg_local->loc1,
-					   0);
-		}
-	}
-
-	/* generation number matches, self heal already done or
-	 * self heal done in background: just do STACK_UNWIND 
-	 */
-	tmp_inode = local->loc1.inode;
-	tmp_dict = local->dict;
-
-	unify_local_wipe (local);
-
-	/* This is lookup_cbk ()'s UNWIND. */
-	STACK_UNWIND (frame,
-		      local->op_ret,
-		      local->op_errno,
-		      tmp_inode,
-		      &local->stbuf,
-		      local->dict,
-                      &local->oldpostparent);
-
-	if (tmp_dict)
-		dict_unref (tmp_dict);
-
-	return 0;
-}
-
diff --git a/xlators/cluster/unify/src/unify.c b/xlators/cluster/unify/src/unify.c
deleted file mode 100644
index f3da6d0ba..000000000
--- a/xlators/cluster/unify/src/unify.c
+++ /dev/null
@@ -1,4589 +0,0 @@
-/*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-/**
- * xlators/cluster/unify:
- *     - This xlator is one of the main translator in GlusterFS, which
- *   actually does the clustering work of the file system. One need to 
- *   understand that, unify assumes file to be existing in only one of 
- *   the child node, and directories to be present on all the nodes. 
- *
- * NOTE:
- *   Now, unify has support for global namespace, which is used to keep a 
- * global view of fs's namespace tree. The stat for directories are taken
- * just from the namespace, where as for files, just 'ia_ino' is taken from
- * Namespace node, and other stat info is taken from the actual storage node.
- * Also Namespace node helps to keep consistant inode for files across 
- * glusterfs (re-)mounts.
- */
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "unify.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "stack.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include <signal.h>
-#include <libgen.h>
-#include "compat-errno.h"
-#include "compat.h"
-
-#define UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR(_loc) do { \
-  if (!(_loc && _loc->inode)) {                            \
-    STACK_UNWIND (frame, -1, EINVAL, NULL, NULL, NULL);    \
-    return 0;                                              \
-  }                                                        \
-} while(0)
-
-
-#define UNIFY_CHECK_FD_CTX_AND_UNWIND_ON_ERR(_fd) do { \
-  if (!(_fd && !fd_ctx_get (_fd, this, NULL))) {       \
-    STACK_UNWIND (frame, -1, EBADFD, NULL, NULL);      \
-    return 0;                                          \
-  }                                                    \
-} while(0)
-
-#define UNIFY_CHECK_FD_AND_UNWIND_ON_ERR(_fd) do { \
-  if (!_fd) {                                      \
-    STACK_UNWIND (frame, -1, EBADFD, NULL, NULL);  \
-    return 0;                                      \
-  }                                                \
-} while(0)
-
-/**
- * unify_local_wipe - free all the extra allocation of local->* here.
- */
-static void 
-unify_local_wipe (unify_local_t *local)
-{
-	/* Free the strdup'd variables in the local structure */
-	if (local->name) {
-		GF_FREE (local->name);
-	}
-	loc_wipe (&local->loc1);
-	loc_wipe (&local->loc2);
-}
-
-
-
-/*
- * unify_normalize_stats -
- */
-void
-unify_normalize_stats (struct statvfs *buf,
-		       unsigned long bsize,
-		       unsigned long frsize)
-{
-	double factor;
-
-	if (buf->f_bsize != bsize) {
-		factor = ((double) buf->f_bsize) / bsize;
-		buf->f_bsize  = bsize;
-		buf->f_bfree  = (fsblkcnt_t) (factor * buf->f_bfree);
-		buf->f_bavail = (fsblkcnt_t) (factor * buf->f_bavail);
-	}
-  
-	if (buf->f_frsize != frsize) {
-		factor = ((double) buf->f_frsize) / frsize;
-		buf->f_frsize = frsize;
-		buf->f_blocks = (fsblkcnt_t) (factor * buf->f_blocks);
-	}
-}
-
-
-xlator_t *
-unify_loc_subvol (loc_t *loc, xlator_t *this)
-{
-	unify_private_t *priv = NULL;
-	xlator_t        *subvol = NULL;
-	int16_t         *list = NULL;
-	long             index = 0;
-	xlator_t        *subvol_i = NULL;
-	int              ret = 0;
-	uint64_t         tmp_list = 0;
-
-	priv   = this->private;
-	subvol = NS (this);
-
-	if (!IA_ISDIR (loc->inode->ia_type)) {
-		ret = inode_ctx_get (loc->inode, this, &tmp_list);
-		list = (int16_t *)(long)tmp_list;
-		if (!list)
-			goto out;
-
-		for (index = 0; list[index] != -1; index++) {
-			subvol_i = priv->xl_array[list[index]];
-			if (subvol_i != NS (this)) {
-				subvol = subvol_i;
-				break;
-			}
-		}
-	}
-out:
-	return subvol;
-}
-
-
-
-/**
- * unify_statfs_cbk -
- */
-int32_t
-unify_statfs_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  struct statvfs *stbuf)
-{
-	int32_t callcnt = 0;
-	struct statvfs *dict_buf = NULL;
-	unsigned long bsize;
-	unsigned long frsize;
-	unify_local_t *local = (unify_local_t *)frame->local;
-	call_frame_t *prev_frame = cookie;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret >= 0) {
-			/* when a call is successfull, add it to local->dict */
-			dict_buf = &local->statvfs_buf;
-
-			if (dict_buf->f_bsize != 0) {
-				bsize  = max (dict_buf->f_bsize, 
-					      stbuf->f_bsize);
-
-				frsize = max (dict_buf->f_frsize, 
-					      stbuf->f_frsize);
-				unify_normalize_stats(dict_buf, bsize, frsize);
-				unify_normalize_stats(stbuf, bsize, frsize);
-			} else {
-				dict_buf->f_bsize   = stbuf->f_bsize;
-				dict_buf->f_frsize  = stbuf->f_frsize;
-			}
-      
-			dict_buf->f_blocks += stbuf->f_blocks;
-			dict_buf->f_bfree  += stbuf->f_bfree;
-			dict_buf->f_bavail += stbuf->f_bavail;
-			dict_buf->f_files  += stbuf->f_files;
-			dict_buf->f_ffree  += stbuf->f_ffree;
-			dict_buf->f_favail += stbuf->f_favail;
-			dict_buf->f_fsid    = stbuf->f_fsid;
-			dict_buf->f_flag    = stbuf->f_flag;
-			dict_buf->f_namemax = stbuf->f_namemax;
-			local->op_ret = op_ret;
-		} else {
-			/* fop on storage node has failed due to some error */
-			if (op_errno != ENOTCONN) {
-				gf_log (this->name, GF_LOG_ERROR, 
-					"child(%s): %s", 
-					prev_frame->this->name, 
-					strerror (op_errno));
-			}
-			local->op_errno = op_errno;
-		}
-		callcnt = --local->call_count;
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		STACK_UNWIND (frame, local->op_ret, local->op_errno, 
-			      &local->statvfs_buf);
-	}
-
-	return 0;
-}
-
-/**
- * unify_statfs -
- */
-int32_t
-unify_statfs (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc)
-{
-	unify_local_t *local = NULL;
-	xlator_list_t *trav = this->children;
-
-	INIT_LOCAL (frame, local);
-	local->call_count = ((unify_private_t *)this->private)->child_count;
-
-	while(trav) {
-		STACK_WIND (frame,
-			    unify_statfs_cbk,
-			    trav->xlator,
-			    trav->xlator->fops->statfs,
-			    loc);
-		trav = trav->next;
-	}
-
-	return 0;
-}
-
-/**
- * unify_buf_cbk - 
- */
-int32_t
-unify_buf_cbk (call_frame_t *frame,
-	       void *cookie,
-	       xlator_t *this,
-	       int32_t op_ret,
-	       int32_t op_errno,
-	       struct iatt *buf)
-{
-	int32_t callcnt = 0;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-	call_frame_t *prev_frame = cookie;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-    
-		if (op_ret == -1) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"%s(): child(%s): path(%s): %s", 
-				gf_fop_list[frame->root->op],
-				prev_frame->this->name, 
-				(local->loc1.path)?local->loc1.path:"", 
-				strerror (op_errno));
-
-			local->op_errno = op_errno;
-			if ((op_errno == ENOENT) && priv->optimist) 
-				local->op_ret = 0;
-		}
-
-		if (op_ret >= 0) {
-			local->op_ret = 0;
-
-			if (NS (this) == prev_frame->this) {
-				local->ia_ino = buf->ia_ino;
-				/* If the entry is directory, get the stat
-				   from NS node */
-				if (IA_ISDIR (buf->ia_type) || 
-				    !local->stbuf.ia_blksize) {
-					local->stbuf = *buf;
-				}
-			}
-
-			if ((!IA_ISDIR (buf->ia_type)) && 
-			    (NS (this) != prev_frame->this)) {
-				/* If file, take the stat info from Storage 
-				   node. */
-				local->stbuf = *buf;
-			}
-		}
-	}
-	UNLOCK (&frame->lock);
-    
-	if (!callcnt) {
-		/* If the inode number is not filled, operation should
-		   fail */
-		if (!local->ia_ino)
-			local->op_ret = -1;
-
-		local->stbuf.ia_ino = local->ia_ino;
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, local->op_ret, local->op_errno, 
-			      &local->stbuf);
-	}
-
-	return 0;
-}
-
-#define check_if_dht_linkfile(s) \
-        ((st_mode_from_ia (s->ia_prot, s->ia_type) & ~S_IFMT) == S_ISVTX)
-
-/**
- * unify_lookup_cbk - 
- */
-int32_t 
-unify_lookup_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  inode_t *inode,
-		  struct iatt *buf,
-		  dict_t *dict,
-                  struct iatt *postparent)
-{
-	int32_t callcnt = 0;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-	inode_t *tmp_inode = NULL;
-	dict_t *local_dict = NULL;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
- 
-		if (op_ret == -1) {
-                        if (local->revalidate &&
-                            (op_errno == ESTALE)) {
-                                /* ESTALE takes priority */
-                                local->op_errno = op_errno;
-                                local->failed = 1;
-                        }
-
-			if ((op_errno != ENOTCONN) 
-                            && (op_errno != ENOENT)
-                            && (local->op_errno != ESTALE)) {
-                                /* if local->op_errno is already ESTALE, then
-                                 * ESTALE has to propogated to the parent first.
-                                 * do not enter here.
-                                 */
-				gf_log (this->name, GF_LOG_ERROR,
-					"child(%s): path(%s): %s", 
-					priv->xl_array[(long)cookie]->name, 
-					local->loc1.path, strerror (op_errno));
-                                local->op_errno = op_errno;
-				local->failed = 1;
-
-			} else if (local->revalidate && 
-                                   (local->op_errno != ESTALE) &&
-				   !(priv->optimist && (op_errno == ENOENT))) {
-
-				gf_log (this->name, 
-					(op_errno == ENOTCONN) ? 
-					GF_LOG_DEBUG:GF_LOG_ERROR,
-					"child(%s): path(%s): %s", 
-					priv->xl_array[(long)cookie]->name, 
-					local->loc1.path, strerror (op_errno));
-				local->op_errno = op_errno;
-				local->failed = 1;
-			}
-		}
-
-		if (op_ret == 0) {
-			local->op_ret = 0; 
-			
-			if (check_if_dht_linkfile(buf)) {
-				gf_log (this->name, GF_LOG_CRITICAL,
-					"file %s may be DHT link file on %s, "
-					"make sure the backend is not shared "
-					"between unify and DHT", 
-					local->loc1.path, 
-					priv->xl_array[(long)cookie]->name);
-			}
-
-			if (local->stbuf.ia_type && local->stbuf.ia_blksize) {
-				/* make sure we already have a stbuf
-				   stored in local->stbuf */
-				if (IA_ISDIR (local->stbuf.ia_type) && 
-				    !IA_ISDIR (buf->ia_type)) {
-					gf_log (this->name, GF_LOG_CRITICAL, 
-						"[CRITICAL] '%s' is directory "
-						"on namespace, non-directory "
-						"on node '%s', returning EIO",
-						local->loc1.path, 
-						priv->xl_array[(long)cookie]->name);
-					local->return_eio = 1;
-				}
-				if (!IA_ISDIR (local->stbuf.ia_type) && 
-				    IA_ISDIR (buf->ia_type)) {
-					gf_log (this->name, GF_LOG_CRITICAL, 
-						"[CRITICAL] '%s' is directory "
-						"on node '%s', non-directory "
-						"on namespace, returning EIO",
-						local->loc1.path, 
-						priv->xl_array[(long)cookie]->name);
-					local->return_eio = 1;
-				}
-			}
-	
-			if (!local->revalidate && !IA_ISDIR (buf->ia_type)) {
-				/* This is the first time lookup on file*/
-				if (!local->list) {
-					/* list is not allocated, allocate 
-					   the max possible range */
-					local->list = GF_CALLOC (1, 2 * (priv->child_count + 2),
-                                                                 gf_unify_mt_int16_t);
-					if (!local->list) {
-						gf_log (this->name, 
-							GF_LOG_CRITICAL, 
-							"Not enough memory");
-						STACK_UNWIND (frame, -1, 
-							      ENOMEM, inode, 
-							      NULL, NULL, NULL);
-						return 0;
-					}
-				}
-				/* update the index of the list */
-				local->list [local->index++] = 
-					(int16_t)(long)cookie;
-			}
-                        
-                        if (!local->revalidate && IA_ISDIR (buf->ia_type)) {
-                                /* fresh lookup of a directory */
-                                inode_ctx_put (local->loc1.inode, this, 
-                                               priv->inode_generation);
-                        }
-
-			if ((!local->dict) && dict &&
-			    (priv->xl_array[(long)cookie] != NS(this)))	{
-				local->dict = dict_ref (dict);
-			}
-
-			/* index of NS node is == total child count */
-			if (priv->child_count == (int16_t)(long)cookie) {
-				/* Take the inode number from namespace */
-				local->ia_ino = buf->ia_ino;
-				if (IA_ISDIR (buf->ia_type) || 
-				    !(local->stbuf.ia_blksize)) {
-					local->stbuf = *buf;
-                                        local->oldpostparent = *postparent;
-				}
-			} else if (!IA_ISDIR (buf->ia_type)) {
-				/* If file, then get the stat from 
-				   storage node */
-				local->stbuf = *buf;
-			}
-
-			if (local->ia_nlink < buf->ia_nlink) {
-				local->ia_nlink = buf->ia_nlink;
-			}
-		}
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		local_dict = local->dict;
-		if (local->return_eio) {
-			gf_log (this->name, GF_LOG_CRITICAL, 
-				"[CRITICAL] Unable to fix the path (%s) with "
-				"self-heal, try manual verification. "
-				"returning EIO.", local->loc1.path);
-			unify_local_wipe (local);
-			STACK_UNWIND (frame, -1, EIO, inode, NULL, NULL);
-			if (local_dict)	{
-				dict_unref (local_dict);
-			}
-			return 0;
-		}
-
-		if (!local->stbuf.ia_blksize) {
-			/* Inode not present */
-			local->op_ret = -1;
-		} else {
-			if (!local->revalidate && 
-			    !IA_ISDIR (local->stbuf.ia_type)) { 
-				/* If its a file, big array is useless, 
-				   allocate the smaller one */
-				int16_t *list = NULL;
-				list = GF_CALLOC (1, 2 * (local->index + 1),
-                                                  gf_unify_mt_int16_t);
-				ERR_ABORT (list);
-				memcpy (list, local->list, 2 * local->index);
-				/* Make the end of the list as -1 */
-				GF_FREE (local->list);
-				local->list = list;
-				local->list [local->index] = -1;
-				/* Update the inode's ctx with proper array */
-				/* TODO: log on failure */
-				inode_ctx_put (local->loc1.inode, this, 
-					       (uint64_t)(long)local->list);
-			}
-
-			if (IA_ISDIR(local->loc1.inode->ia_type)) {
-				/* lookup is done for directory */
-				if (local->failed && priv->self_heal) {
-					/* Triggering self-heal */
-                                        /* means, self-heal required for this 
-					   inode */
-					local->inode_generation = 0; 
-					priv->inode_generation++;
-				}
-			} else {
-				local->stbuf.ia_ino = local->ia_ino;
-			}
-	  
-			local->stbuf.ia_nlink = local->ia_nlink;
-		}
-		if (local->op_ret == -1) {
-			if (!local->revalidate && local->list)
-				GF_FREE (local->list);
-		}
-
-		if ((local->op_ret >= 0) && local->failed && 
-		    local->revalidate) {
-			/* Done revalidate, but it failed */
-			if ((op_errno != ENOTCONN) 
-                            && (local->op_errno != ESTALE)) {
-				gf_log (this->name, GF_LOG_ERROR, 
-					"Revalidate failed for path(%s): %s", 
-					local->loc1.path, strerror (op_errno));
-			}
-			local->op_ret = -1;
-		}
-
-		if ((priv->self_heal && !priv->optimist) && 
-		    (!local->revalidate && (local->op_ret == 0) && 
-		     IA_ISDIR(local->stbuf.ia_type))) {
-			/* Let the self heal be done here */
-			zr_unify_self_heal (frame, this, local);
-			local_dict = NULL;
-		} else {
-                        if (local->failed) {
-                                /* NOTE: directory lookup is sent to all 
-                                 * subvolumes and success from a subvolume
-                                 * might set local->op_ret to 0 (zero) */
-                                local->op_ret = -1;
-                        }
-
-			/* either no self heal, or op_ret == -1 (failure) */
-			tmp_inode = local->loc1.inode;
-			unify_local_wipe (local);
-			STACK_UNWIND (frame, local->op_ret, local->op_errno, 
-				      tmp_inode, &local->stbuf, local->dict,
-                                      &local->oldpostparent);
-		}
-		if (local_dict) {
-			dict_unref (local_dict);
-		}
-	}
-  
-	return 0;
-}
-
-/**
- * unify_lookup - 
- */
-int32_t 
-unify_lookup (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc,
-	      dict_t *xattr_req)
-{
-	unify_local_t *local = NULL;
-	unify_private_t *priv = this->private;
-	int16_t *list = NULL;
-	long index = 0;
-
-	if (!(loc && loc->inode)) {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"%s: Argument not right", loc?loc->path:"(null)");
-		STACK_UNWIND (frame, -1, EINVAL, NULL, NULL, NULL, NULL);
-		return 0;
-	}
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	loc_copy (&local->loc1, loc);
-	if (local->loc1.path == NULL) {
-		gf_log (this->name, GF_LOG_CRITICAL, "Not enough memory :O");
-		STACK_UNWIND (frame, -1, ENOMEM, loc->inode, NULL, NULL, NULL);
-		return 0;
-	}
-        
-        if (inode_ctx_get (loc->inode, this, NULL)
-            && IA_ISDIR (loc->inode->ia_type)) {
-                local->revalidate = 1;
-        }
-
-	if (!inode_ctx_get (loc->inode, this, NULL) && 
-	    loc->inode->ia_type && 
-	    !IA_ISDIR (loc->inode->ia_type)) {
-		uint64_t tmp_list = 0;
-		/* check if revalidate or fresh lookup */
-		inode_ctx_get (loc->inode, this, &tmp_list);
-		local->list = (int16_t *)(long)tmp_list;
-	}
-
-	if (local->list) {
-		list = local->list;
-		for (index = 0; list[index] != -1; index++);
-		if (index != 2) {
-			if (index < 2) {
-				gf_log (this->name, GF_LOG_ERROR,
-					"returning ESTALE for %s: file "
-					"count is %ld", loc->path, index);
-				/* Print where all the file is present */
-				for (index = 0; 
-				     local->list[index] != -1; index++) {
-					gf_log (this->name, GF_LOG_ERROR, 
-						"%s: found on %s", loc->path, 
-						priv->xl_array[list[index]]->name);
-				}
-				unify_local_wipe (local);
-				STACK_UNWIND (frame, -1, ESTALE, 
-					      NULL, NULL, NULL, NULL);
-				return 0;  
-			} else {
-				/* There are more than 2 presences */
-				/* Just log and continue */
-				gf_log (this->name, GF_LOG_ERROR,
-					"%s: file count is %ld", 
-					loc->path, index);
-				/* Print where all the file is present */
-				for (index = 0; 
-				     local->list[index] != -1; index++) {
-					gf_log (this->name, GF_LOG_ERROR, 
-						"%s: found on %s", loc->path, 
-						priv->xl_array[list[index]]->name);
-				}
-			}
-		}
-      
-		/* is revalidate */
-		local->revalidate = 1;
-      
-		for (index = 0; list[index] != -1; index++)
-			local->call_count++;
-      
-		for (index = 0; list[index] != -1; index++) {
-			char need_break = (list[index+1] == -1);
-			STACK_WIND_COOKIE (frame,
-					   unify_lookup_cbk,
-					   (void *)(long)list[index], //cookie
-					   priv->xl_array [list[index]],
-					   priv->xl_array [list[index]]->fops->lookup,
-					   loc,
-					   xattr_req);
-			if (need_break)
-				break;
-		}
-	} else {
-		if (loc->inode->ia_type) {
-			if (inode_ctx_get (loc->inode, this, NULL)) {
-				inode_ctx_get (loc->inode, this, 
-					       &local->inode_generation);
-			}
-		}
-		/* This is first call, there is no list */
-		/* call count should be all child + 1 namespace */
-		local->call_count = priv->child_count + 1;
-      
-		for (index = 0; index <= priv->child_count; index++) {
-			STACK_WIND_COOKIE (frame,
-					   unify_lookup_cbk,
-					   (void *)index, //cookie
-					   priv->xl_array[index],
-					   priv->xl_array[index]->fops->lookup,
-					   loc,
-					   xattr_req);
-		}
-	}
-
-	return 0;
-}
-
-/**
- * unify_stat - if directory, get the stat directly from NameSpace child.
- *     if file, check for a hint and send it only there (also to NS).
- *     if its a fresh stat, then do it on all the nodes.
- *
- * NOTE: for all the call, sending cookie as xlator pointer, which will be 
- *       used in cbk.
- */
-int32_t
-unify_stat (call_frame_t *frame,
-	    xlator_t *this,
-	    loc_t *loc)
-{
-	unify_local_t *local = NULL;
-	unify_private_t *priv = this->private;
-	int16_t index = 0;
-	int16_t *list = NULL;
-	uint64_t tmp_list = 0;
-
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	loc_copy (&local->loc1, loc);
-	if (local->loc1.path == NULL) {
-		gf_log (this->name, GF_LOG_CRITICAL, "Not enough memory :O");
-		STACK_UNWIND (frame, -1, ENOMEM, NULL);
-		return 0;
-	}
-	local->ia_ino = loc->inode->ino;
-	if (IA_ISDIR (loc->inode->ia_type)) {
-		/* Directory */
-		local->call_count = 1;
-		STACK_WIND (frame, unify_buf_cbk, NS(this),
-			    NS(this)->fops->stat, loc);
-	} else {
-		/* File */
-		inode_ctx_get (loc->inode, this, &tmp_list);
-    		list = (int16_t *)(long)tmp_list;
-
-		for (index = 0; list[index] != -1; index++)
-			local->call_count++;
-    
-		for (index = 0; list[index] != -1; index++) {
-			char need_break = (list[index+1] == -1);
-			STACK_WIND (frame,
-				    unify_buf_cbk,
-				    priv->xl_array[list[index]],
-				    priv->xl_array[list[index]]->fops->stat,
-				    loc);
-			if (need_break)
-				break;
-		}
-	}
-
-	return 0;
-}
-
-/**
- * unify_access_cbk -
- */
-int32_t
-unify_access_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno)
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-
-/**
- * unify_access - Send request to only namespace, which has all the 
- *      attributes set for the file.
- */
-int32_t
-unify_access (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc,
-	      int32_t mask)
-{
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-
-	STACK_WIND (frame,
-		    unify_access_cbk,
-		    NS(this),
-		    NS(this)->fops->access,
-		    loc,
-		    mask);
-
-	return 0;
-}
-
-int32_t
-unify_mkdir_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 inode_t *inode,
-                 struct iatt *buf,
-                 struct iatt *preparent,
-                 struct iatt *postparent)
-{
-	int32_t callcnt = 0;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-	inode_t *tmp_inode = NULL;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-  
-		if ((op_ret == -1) && !(priv->optimist && 
-					(op_errno == ENOENT || 
-					 op_errno == EEXIST))) {
-			/* TODO: Decrement the inode_generation of 
-			 * this->inode's parent inode, hence the missing 
-			 * directory is created properly by self-heal. 
-			 * Currently, there is no way to get the parent 
-			 * inode directly.
-			 */
-			gf_log (this->name, GF_LOG_ERROR,
-				"child(%s): path(%s): %s", 
-				priv->xl_array[(long)cookie]->name, 
-				local->loc1.path, strerror (op_errno));
-			if (op_errno != EEXIST)
-				local->failed = 1;
-			local->op_errno = op_errno;
-		}
-  
-		if (op_ret >= 0)
-			local->op_ret = 0;
-
-	}
-	UNLOCK (&frame->lock);
-  
-	if (!callcnt) {
-		if (!local->failed) {
-			inode_ctx_put (local->loc1.inode, this, 
-				       priv->inode_generation);
-		}
-		
-		tmp_inode = local->loc1.inode;
-		unify_local_wipe (local);
-
-		STACK_UNWIND (frame, local->op_ret, local->op_errno, 
-			      tmp_inode, &local->stbuf,
-                              &local->oldpreparent, &local->oldpostparent);
-	}
-
-	return 0;
-}
-
-/**
- * unify_ns_mkdir_cbk -
- */
-int32_t
-unify_ns_mkdir_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    inode_t *inode,
-                    struct iatt *buf,
-                    struct iatt *preparent,
-                    struct iatt *postparent)
-{
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-	long index = 0;
-
-	if (op_ret == -1) {
-		/* No need to send mkdir request to other servers, 
-		 * as namespace action failed 
-		 */
-		gf_log (this->name, GF_LOG_ERROR,
-			"namespace: path(%s): %s", 
-			local->name, strerror (op_errno));
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, op_ret, op_errno, inode, NULL,
-                              NULL, NULL);
-		return 0;
-	}
-  
-	/* Create one inode for this entry */
-	local->op_ret = 0;
-	local->stbuf = *buf;
-
-        local->oldpreparent = *preparent;
-        local->oldpostparent = *postparent;
-
-	local->call_count = priv->child_count;
-
-	/* Send mkdir request to all the nodes now */
-	for (index = 0; index < priv->child_count; index++) {
-		STACK_WIND_COOKIE (frame,
-				   unify_mkdir_cbk,
-				   (void *)index, //cookie
-				   priv->xl_array[index],
-				   priv->xl_array[index]->fops->mkdir,
-				   &local->loc1,
-				   local->mode);
-	}
-  
-	return 0;
-}
-
-
-/**
- * unify_mkdir -
- */
-int32_t
-unify_mkdir (call_frame_t *frame,
-	     xlator_t *this,
-	     loc_t *loc,
-	     mode_t mode)
-{
-	unify_local_t *local = NULL;
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	local->mode = mode;
-
-	loc_copy (&local->loc1, loc);
-
-	if (local->loc1.path == NULL) {
-		gf_log (this->name, GF_LOG_CRITICAL, "Not enough memory :O");
-		STACK_UNWIND (frame, -1, ENOMEM, NULL, NULL);
-		return 0;
-	}
-
-	STACK_WIND (frame,
-		    unify_ns_mkdir_cbk,
-		    NS(this),
-		    NS(this)->fops->mkdir,
-		    loc,
-		    mode);
-	return 0;
-}
-
-/**
- * unify_rmdir_cbk -
- */
-int32_t
-unify_rmdir_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-                 struct iatt *preparent,
-                 struct iatt *postparent)
-{
-	int32_t callcnt = 0;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-		if (op_ret == 0 || (priv->optimist && (op_errno == ENOENT)))
-			local->op_ret = 0;
-		if (op_ret == -1)
-			local->op_errno = op_errno;
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, local->op_ret, local->op_errno,
-                              &local->oldpreparent, &local->oldpostparent);
-	}
-
-	return 0;
-}
-
-/**
- * unify_ns_rmdir_cbk -
- */
-int32_t
-unify_ns_rmdir_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-                    struct iatt *preparent,
-                    struct iatt *postparent)
-{
-	int16_t index = 0;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-  
-	if (op_ret == -1) {
-		/* No need to send rmdir request to other servers, 
-		 * as namespace action failed 
-		 */
-		gf_log (this->name, 
-			((op_errno != ENOTEMPTY) ? 
-			 GF_LOG_ERROR : GF_LOG_DEBUG),
-			"namespace: path(%s): %s", 
-			local->loc1.path, strerror (op_errno));
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, op_ret, op_errno, NULL, NULL);
-		return 0;
-	}
-
-	local->call_count = priv->child_count;
-
-        local->oldpreparent = *preparent;
-        local->oldpostparent = *postparent;
-
-	for (index = 0; index < priv->child_count; index++) {
-		STACK_WIND (frame,
-			    unify_rmdir_cbk,
-			    priv->xl_array[index],
-			    priv->xl_array[index]->fops->rmdir,
-			    &local->loc1);
-	}
-
-	return 0;
-}
-
-/**
- * unify_rmdir -
- */
-int32_t
-unify_rmdir (call_frame_t *frame,
-	     xlator_t *this,
-	     loc_t *loc)
-{
-	unify_local_t *local = NULL;
-
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-
-	loc_copy (&local->loc1, loc);
-	if (local->loc1.path == NULL) {
-		gf_log (this->name, GF_LOG_CRITICAL, "Not enough memory :O");
-		STACK_UNWIND (frame, -1, ENOMEM, NULL, NULL);
-		return 0;
-	}
-
-	STACK_WIND (frame,
-		    unify_ns_rmdir_cbk,
-		    NS(this),
-		    NS(this)->fops->rmdir,
-		    loc);
-
-	return 0;
-}
-
-/**
- * unify_open_cbk -
- */
-int32_t
-unify_open_cbk (call_frame_t *frame,
-		void *cookie,
-		xlator_t *this,
-		int32_t op_ret,
-		int32_t op_errno,
-		fd_t *fd)
-{
-	int32_t callcnt = 0;
-	unify_local_t *local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret >= 0) {
-			local->op_ret = op_ret;
-			if (NS(this) != (xlator_t *)cookie) {
-				/* Store child node's ptr, used in 
-				   all the f*** / FileIO calls */
-				fd_ctx_set (fd, this, (uint64_t)(long)cookie);
-			}
-		}
-		if (op_ret == -1) {
-			local->op_errno = op_errno;
-			local->failed = 1;
-		}
-		callcnt = --local->call_count;
-	}
-	UNLOCK (&frame->lock);
-  
-	if (!callcnt) {
-		if ((local->failed == 1) && (local->op_ret >= 0)) {
-			local->call_count = 1;
-			/* return -1 to user */
-			local->op_ret = -1;
-			//local->op_errno = EIO; 
-      
-			if (!fd_ctx_get (local->fd, this, NULL)) {
-				gf_log (this->name, GF_LOG_ERROR, 
-					"Open success on child node, "
-					"failed on namespace");
-			} else {
-				gf_log (this->name, GF_LOG_ERROR, 
-					"Open success on namespace, "
-					"failed on child node");
-			}
-		}
-
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, local->op_ret, 
-			      local->op_errno, local->fd);
-	}
-
-	return 0;
-}
-
-#ifdef GF_DARWIN_HOST_OS
-/**
- * unify_create_lookup_cbk - 
- */
-int32_t 
-unify_open_lookup_cbk (call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       inode_t *inode,
-		       struct iatt *buf,
-		       dict_t *dict,
-		       struct iatt *postparent)
-{
-	int32_t callcnt = 0;
-	int16_t index = 0;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-		if ((op_ret == -1) && (op_errno != ENOENT)) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"child(%s): path(%s): %s", 
-				priv->xl_array[(long)cookie]->name, 
-				local->loc1.path, strerror (op_errno));
-			local->op_errno = op_errno;
-		}
-    
-		if (op_ret >= 0) {
-			local->op_ret = op_ret; 
-			local->index++;
-			if (NS(this) == priv->xl_array[(long)cookie]) {
-				local->list[0] = (int16_t)(long)cookie;
-			} else {
-				local->list[1] = (int16_t)(long)cookie;
-			}
-			if (IA_ISDIR (buf->ia_type))
-				local->failed = 1;
-		}
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		int16_t file_list[3] = {0,};
-		local->op_ret = -1;
-
-		file_list[0] = local->list[0];
-		file_list[1] = local->list[1];
-		file_list[2] = -1;
-
-		if (local->index != 2) {
-			/* Lookup failed, can't do open */
-			gf_log (this->name, GF_LOG_ERROR,
-				"%s: present on %d nodes", 
-				local->name, local->index);
-
-			if (local->index < 2) {
-				unify_local_wipe (local);
-				gf_log (this->name, GF_LOG_ERROR,
-					"returning as file found on less "
-					"than 2 nodes");
-				STACK_UNWIND (frame, local->op_ret, 
-					      local->op_errno, local->fd);
-				return 0;
-			}
-		}
-
-		if (local->failed) {
-			/* Open on directory, return EISDIR */
-			unify_local_wipe (local);
-			STACK_UNWIND (frame, -1, EISDIR, local->fd);
-			return 0;
-		}
-
-		/* Everything is perfect :) */    
-		local->call_count = 2;
-    
-		for (index = 0; file_list[index] != -1; index++) {
-			char need_break = (file_list[index+1] == -1);
-			STACK_WIND_COOKIE (frame,
-					   unify_open_cbk,
-					   priv->xl_array[file_list[index]],
-					   priv->xl_array[file_list[index]],
-					   priv->xl_array[file_list[index]]->fops->open,
-					   &local->loc1,
-					   local->flags,
-					   local->fd, local->wbflags);
-			if (need_break)
-				break;
-		}
-	}
-
-	return 0;
-}
-
-
-int32_t
-unify_open_readlink_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno,
-			 const char *path,
-			 struct iatt *sbuf)
-{
-	int16_t index = 0;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-
-	if (op_ret == -1) {
-		STACK_UNWIND (frame, -1, ENOENT);
-		return 0;
-	}
-
-	if (path[0] == '/') {
-		local->name = gf_strdup (path);
-		ERR_ABORT (local->name);
-	} else {
-		char *tmp_str = gf_strdup (local->loc1.path);
-		char *tmp_base = dirname (tmp_str);
-		local->name = GF_CALLOC (1, PATH_MAX, gf_unify_mt_char);
-		strcpy (local->name, tmp_base);
-		strncat (local->name, "/", 1);
-		strcat (local->name, path);
-		GF_FREE (tmp_str);
-	}
-  
-	local->list = GF_CALLOC (1, sizeof (int16_t) * 3,
-                                 gf_unify_mt_int16_t);
-	ERR_ABORT (local->list);
-	local->call_count = priv->child_count + 1;
-	local->op_ret = -1;
-	for (index = 0; index <= priv->child_count; index++) {
-		/* Send the lookup to all the nodes including namespace */
-		STACK_WIND_COOKIE (frame,
-				   unify_open_lookup_cbk,
-				   (void *)(long)index,
-				   priv->xl_array[index],
-				   priv->xl_array[index]->fops->lookup,
-				   &local->loc1,
-				   NULL);
-	}
-
-	return 0;
-}
-#endif /* GF_DARWIN_HOST_OS */
-
-/**
- * unify_open - 
- */
-int32_t
-unify_open (call_frame_t *frame,
-	    xlator_t *this,
-	    loc_t *loc,
-	    int32_t flags,
-	    fd_t *fd,
-            int32_t wbflags)
-{
-	unify_private_t *priv = this->private;
-	unify_local_t *local = NULL;
-	int16_t *list = NULL;
-	int16_t index = 0;
-	int16_t file_list[3] = {0,};
-	uint64_t tmp_list = 0;
-
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-
-	/* Init */
-	INIT_LOCAL (frame, local);
-	loc_copy (&local->loc1, loc);
-	local->fd    = fd;
-	local->flags = flags;
-        local->wbflags = wbflags;
-	inode_ctx_get (loc->inode, this, &tmp_list);
-	list = (int16_t *)(long)tmp_list;
-
-	local->list = list;
-	file_list[0] = priv->child_count; /* Thats namespace */
-	file_list[2] = -1;
-	for (index = 0; list[index] != -1; index++) {
-		local->call_count++;
-		if (list[index] != priv->child_count)
-			file_list[1] = list[index];
-	}
-
-	if (local->call_count != 2) {
-		/* If the lookup was done for file */
-		gf_log (this->name, GF_LOG_ERROR,
-			"%s: entry_count is %d",
-			loc->path, local->call_count);
-		for (index = 0; local->list[index] != -1; index++)
-			gf_log (this->name, GF_LOG_ERROR, "%s: found on %s",
-				loc->path, priv->xl_array[list[index]]->name);
-
-		if (local->call_count < 2) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"returning EIO as file found on onlyone node");
-			STACK_UNWIND (frame, -1, EIO, fd);
-			return 0;
-		}
-	}
-
-#ifdef GF_DARWIN_HOST_OS
-	/* Handle symlink here */
-	if (IA_ISLNK (loc->inode->ia_type)) {
-		/* Callcount doesn't matter here */
-		STACK_WIND (frame,
-			    unify_open_readlink_cbk,
-			    NS(this),
-			    NS(this)->fops->readlink,
-			    loc, PATH_MAX);
-		return 0;
-	}
-#endif /* GF_DARWIN_HOST_OS */
-
-	local->call_count = 2;
-	for (index = 0; file_list[index] != -1; index++) {
-		char need_break = (file_list[index+1] == -1);
-		STACK_WIND_COOKIE (frame,
-				   unify_open_cbk,
-				   priv->xl_array[file_list[index]], //cookie
-				   priv->xl_array[file_list[index]],
-				   priv->xl_array[file_list[index]]->fops->open,
-				   loc,
-				   flags,
-				   fd, wbflags);
-		if (need_break)
-			break;
-	}
-
-	return 0;
-}
-
-
-int32_t 
-unify_create_unlink_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno,
-                         struct iatt *preparent,
-                         struct iatt *postparent)
-{
-	unify_local_t *local = frame->local;
-	inode_t *inode = local->loc1.inode;
-
-	unify_local_wipe (local);
-
-	STACK_UNWIND (frame, local->op_ret, local->op_errno, local->fd, 
-		      inode, &local->stbuf,
-                      &local->oldpreparent, &local->oldpostparent);
-  
-	return 0;
-}
-
-/**
- * unify_create_open_cbk -
- */
-int32_t
-unify_create_open_cbk (call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       fd_t *fd)
-{
-	int ret = 0;
-	int32_t callcnt = 0;
-	unify_local_t *local = frame->local;
-	inode_t *inode = NULL;
-	xlator_t *child = NULL;
-	uint64_t tmp_value = 0;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret >= 0) {
-			local->op_ret = op_ret;
-			if (NS(this) != (xlator_t *)cookie) {
-				/* Store child node's ptr, used in all 
-				   the f*** / FileIO calls */
-				/* TODO: log on failure */
-				ret = fd_ctx_get (fd, this, &tmp_value);
-				cookie = (void *)(long)tmp_value;
-			} else {
-				/* NOTE: open successful on namespace.
-				 *       fd's ctx can be used to identify open 
-				 *       failure on storage subvolume. cool 
-				 *       ide ;) */
-				local->failed = 0;
-			}
-		} else {
-			gf_log (this->name, GF_LOG_ERROR,
-				"child(%s): path(%s): %s", 
-				((xlator_t *)cookie)->name,
-				local->loc1.path, strerror (op_errno));
-			local->op_errno = op_errno;
-			local->failed = 1;
-		}
-		callcnt = --local->call_count;
-	}
-	UNLOCK (&frame->lock);
-  
-	if (!callcnt) {
-		if (local->failed == 1 && (local->op_ret >= 0)) {
-			local->call_count = 1;
-			/* return -1 to user */
-			local->op_ret = -1;
-			local->op_errno = EIO;
-			local->fd = fd;
-			local->call_count = 1;
-
-			if (!fd_ctx_get (local->fd, this, &tmp_value)) {
-				child = (xlator_t *)(long)tmp_value;
-
-				gf_log (this->name, GF_LOG_ERROR, 
-					"Create success on child node, "
-					"failed on namespace");
-
-				STACK_WIND (frame,
-					    unify_create_unlink_cbk,
-					    child,
-					    child->fops->unlink,
-					    &local->loc1);
-			} else {
-				gf_log (this->name, GF_LOG_ERROR, 
-					"Create success on namespace, "
-					"failed on child node");
-
-				STACK_WIND (frame,
-					    unify_create_unlink_cbk,
-					    NS(this),
-					    NS(this)->fops->unlink,
-					    &local->loc1);
-			}
-			return 0;
-		}
-		inode = local->loc1.inode;
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, local->op_ret, local->op_errno, fd,
-			      inode, &local->stbuf,
-                              &local->oldpreparent, &local->oldpostparent);
-	}
-	return 0;
-}
-
-/**
- * unify_create_lookup_cbk - 
- */
-int32_t 
-unify_create_lookup_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno,
-			 inode_t *inode,
-			 struct iatt *buf,
-			 dict_t *dict,
-                         struct iatt *postparent)
-{
-	int32_t callcnt = 0;
-	int16_t index = 0;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-		if (op_ret == -1) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"child(%s): path(%s): %s", 
-				priv->xl_array[(long)cookie]->name, 
-				local->loc1.path, strerror (op_errno));
-			local->op_errno = op_errno;
-			local->failed = 1;
-		}
-
-		if (op_ret >= 0) {
-			local->op_ret = op_ret; 
-			local->list[local->index++] = (int16_t)(long)cookie;
-			if (NS(this) == priv->xl_array[(long)cookie]) {
-				local->ia_ino = buf->ia_ino;
-			} else {
-				local->stbuf = *buf;
-			}
-		}
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		int16_t *list = local->list;
-		int16_t file_list[3] = {0,};
-		local->op_ret = -1;
-
-		local->list [local->index] = -1;
-		file_list[0] = list[0];
-		file_list[1] = list[1];
-		file_list[2] = -1;
-
-		local->stbuf.ia_ino = local->ia_ino;
-		/* TODO: log on failure */
-		inode_ctx_put (local->loc1.inode, this, 
-			       (uint64_t)(long)local->list);
-
-		if (local->index != 2) {
-			/* Lookup failed, can't do open */
-			gf_log (this->name, GF_LOG_ERROR,
-				"%s: present on %d nodes", 
-				local->loc1.path, local->index);
-			file_list[0] = priv->child_count;
-			for (index = 0; list[index] != -1; index++) {
-				gf_log (this->name, GF_LOG_ERROR, 
-					"%s: found on %s", local->loc1.path, 
-					priv->xl_array[list[index]]->name);
-				if (list[index] != priv->child_count)
-					file_list[1] = list[index];
-			}
-
-			if (local->index < 2) {
-				unify_local_wipe (local);
-				gf_log (this->name, GF_LOG_ERROR,
-					"returning EIO as file found on "
-					"only one node");
-				STACK_UNWIND (frame, -1, EIO, 
-					      local->fd, inode, NULL,
-                                              NULL, NULL);
-				return 0;
-			}
-		}
-		/* Everything is perfect :) */    
-		local->call_count = 2;
-    
-		for (index = 0; file_list[index] != -1; index++) {
-			char need_break = (file_list[index+1] == -1);
-			STACK_WIND_COOKIE (frame,
-					   unify_create_open_cbk,
-					   priv->xl_array[file_list[index]],
-					   priv->xl_array[file_list[index]],
-					   priv->xl_array[file_list[index]]->fops->open,
-					   &local->loc1,
-					   local->flags,
-					   local->fd, 0);
-			if (need_break)
-				break;
-		}
-	}
-
-	return 0;
-}
-
-
-/**
- * unify_create_cbk -
- */
-int32_t
-unify_create_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  fd_t *fd,
-		  inode_t *inode,
-		  struct iatt *buf,
-                  struct iatt *preparent,
-                  struct iatt *postparent)
-{
-	int ret = 0;
-	unify_local_t *local = frame->local;
-	call_frame_t *prev_frame = cookie;
-	inode_t *tmp_inode = NULL;
-
-	if (op_ret == -1) {
-		/* send unlink () on Namespace */
-		local->op_errno = op_errno;
-		local->op_ret = -1;
-		local->call_count = 1;
-		gf_log (this->name, GF_LOG_ERROR,
-			"create failed on %s (file %s, error %s), "
-			"sending unlink to namespace", 
-			prev_frame->this->name, 
-			local->loc1.path, strerror (op_errno));
-
-		STACK_WIND (frame,
-			    unify_create_unlink_cbk,
-			    NS(this),
-			    NS(this)->fops->unlink,
-			    &local->loc1);
-
-		return 0;
-	}
-
-	if (op_ret >= 0) {
-		local->op_ret = op_ret;
-		local->stbuf = *buf;
-		/* Just inode number should be from NS node */
-		local->stbuf.ia_ino = local->ia_ino;
-
-		/* TODO: log on failure */
-		ret = fd_ctx_set (fd, this, (uint64_t)(long)prev_frame->this);
-	}
-  
-	tmp_inode = local->loc1.inode;
-	unify_local_wipe (local);
-	STACK_UNWIND (frame, local->op_ret, local->op_errno, local->fd, 
-		      tmp_inode, &local->stbuf,
-                      &local->oldpreparent, &local->oldpostparent);
-
-	return 0;
-}
-
-/**
- * unify_ns_create_cbk -
- * 
- */
-int32_t
-unify_ns_create_cbk (call_frame_t *frame,
-		     void *cookie,
-		     xlator_t *this,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     fd_t *fd,
-		     inode_t *inode,
-		     struct iatt *buf,
-                     struct iatt *preparent,
-                     struct iatt *postparent)
-{
-	struct sched_ops *sched_ops = NULL;
-	xlator_t *sched_xl = NULL;
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	int16_t *list = NULL;
-	int16_t index = 0;
-
-	if (op_ret == -1) {
-		/* No need to send create request to other servers, as 
-		   namespace action failed. Handle exclusive create here. */
-		if ((op_errno != EEXIST) || 
-		    ((op_errno == EEXIST) && 
-		     ((local->flags & O_EXCL) == O_EXCL))) {
-			/* If its just a create call without O_EXCL, 
-			   don't do this */
-			gf_log (this->name, GF_LOG_ERROR,
-				"namespace: path(%s): %s", 
-				local->loc1.path, strerror (op_errno));
-			unify_local_wipe (local);
-			STACK_UNWIND (frame, op_ret, op_errno, fd, inode, buf,
-                                      preparent, postparent);
-			return 0;
-		}
-	}
-  
-	if (op_ret >= 0) {
-		/* Get the inode number from the NS node */
-		local->ia_ino = buf->ia_ino;
-  
-                local->oldpreparent = *preparent;
-                local->oldpostparent = *postparent;
-
-		local->op_ret = -1;
-
-		/* Start the mapping list */
-		list = GF_CALLOC (1, sizeof (int16_t) * 3,
-                                  gf_unify_mt_int16_t);
-		ERR_ABORT (list);
-		inode_ctx_put (inode, this, (uint64_t)(long)list);
-		list[0] = priv->child_count;
-		list[2] = -1;
-
-		/* This means, file doesn't exist anywhere in the Filesystem */
-		sched_ops = priv->sched_ops;
-
-		/* Send create request to the scheduled node now */
-		sched_xl = sched_ops->schedule (this, local->loc1.path);
-		if (sched_xl == NULL)
-		{
-			/* send unlink () on Namespace */
-			local->op_errno = ENOTCONN;
-			local->op_ret = -1;
-			local->call_count = 1;
-			gf_log (this->name, GF_LOG_ERROR,
-				"no node online to schedule create:(file %s) "
-				"sending unlink to namespace", 
-				(local->loc1.path)?local->loc1.path:"");
-
-			STACK_WIND (frame,
-				    unify_create_unlink_cbk,
-				    NS(this),
-				    NS(this)->fops->unlink,
-				    &local->loc1);
-	
-			return 0;
-		}
-
-		for (index = 0; index < priv->child_count; index++)
-			if (sched_xl == priv->xl_array[index])
-				break;
-		list[1] = index;
-
-		STACK_WIND (frame, unify_create_cbk,
-			    sched_xl, sched_xl->fops->create,
-			    &local->loc1, local->flags, local->mode, fd);
-	} else {
-		/* File already exists, and there is no O_EXCL flag */
-
-		gf_log (this->name, GF_LOG_DEBUG, 
-			"File(%s) already exists on namespace, sending "
-			"open instead", local->loc1.path);
-
-		local->list = GF_CALLOC (1, sizeof (int16_t) * 3,
-                                         gf_unify_mt_int16_t);
-		ERR_ABORT (local->list);
-		local->call_count = priv->child_count + 1;
-		local->op_ret = -1;
-		for (index = 0; index <= priv->child_count; index++) {
-			/* Send lookup() to all nodes including namespace */
-			STACK_WIND_COOKIE (frame,
-					   unify_create_lookup_cbk,
-					   (void *)(long)index,
-					   priv->xl_array[index],
-					   priv->xl_array[index]->fops->lookup,
-					   &local->loc1,
-					   NULL);
-		}
-	}
-	return 0;
-}
-
-/**
- * unify_create - create a file in global namespace first, so other 
- *    clients can see them. Create the file in storage nodes in background.
- */
-int32_t
-unify_create (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc,
-	      int32_t flags,
-	      mode_t mode,
-	      fd_t *fd)
-{
-	unify_local_t *local = NULL;
-  
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	local->mode = mode;
-	local->flags = flags;
-	local->fd = fd;
-
-	loc_copy (&local->loc1, loc);
-	if (local->loc1.path == NULL) {
-		gf_log (this->name, GF_LOG_CRITICAL, "Not enough memory :O");
-		STACK_UNWIND (frame, -1, ENOMEM, fd, loc->inode, NULL,
-                              NULL, NULL);
-		return 0;
-	}
-
-	STACK_WIND (frame,
-		    unify_ns_create_cbk,
-		    NS(this),
-		    NS(this)->fops->create,
-		    loc,
-		    flags | O_EXCL,
-		    mode,
-		    fd);
-  
-	return 0;
-}
-
-
-/**
- * unify_opendir_cbk - 
- */
-int32_t
-unify_opendir_cbk (call_frame_t *frame,
-		   void *cookie,
-		   xlator_t *this,
-		   int32_t op_ret,
-		   int32_t op_errno,
-		   fd_t *fd)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, fd);
-
-	return 0;
-}
-
-/** 
- * unify_opendir -
- */
-int32_t
-unify_opendir (call_frame_t *frame,
-	       xlator_t *this,
-	       loc_t *loc,
-	       fd_t *fd)
-{
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-
-	STACK_WIND (frame, unify_opendir_cbk,
-		    NS(this), NS(this)->fops->opendir, loc, fd);
-
-	return 0;
-}
-
-
-int32_t
-unify_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *statpre,
-                   struct iatt *statpost)
-{
-	int32_t callcnt = 0;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-	call_frame_t *prev_frame = cookie;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-    
-		if (op_ret == -1) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"%s(): child(%s): path(%s): %s", 
-				gf_fop_list[frame->root->op],
-				prev_frame->this->name, 
-				(local->loc1.path)?local->loc1.path:"", 
-				strerror (op_errno));
-
-			local->op_errno = op_errno;
-			if ((op_errno == ENOENT) && priv->optimist) 
-				local->op_ret = 0;
-		}
-
-		if (op_ret >= 0) {
-			local->op_ret = 0;
-
-			if (NS (this) == prev_frame->this) {
-				local->ia_ino = statpost->ia_ino;
-				/* If the entry is directory, get the stat
-				   from NS node */
-				if (IA_ISDIR (statpost->ia_type) || 
-				    !local->stpost.ia_blksize) {
-					local->stpre = *statpre;
-                                        local->stpost = *statpost;
-				}
-			}
-
-			if ((!IA_ISDIR (statpost->ia_type)) && 
-			    (NS (this) != prev_frame->this)) {
-				/* If file, take the stat info from Storage 
-				   node. */
-				local->stpre = *statpre;
-                                local->stpost = *statpost;
-			}
-		}
-	}
-	UNLOCK (&frame->lock);
-    
-	if (!callcnt) {
-		/* If the inode number is not filled, operation should
-		   fail */
-		if (!local->ia_ino)
-			local->op_ret = -1;
-
-                local->stpre.ia_ino = local->ia_ino;
-		local->stpost.ia_ino = local->ia_ino;
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, local->op_ret, local->op_errno, 
-			      &local->stpre, &local->stpost);
-	}
-
-	return 0;
-}
-
-
-int32_t
-unify_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-               struct iatt *stbuf, int32_t valid)
-{
-	unify_local_t *local = NULL;
-	unify_private_t *priv = this->private;
-	int32_t index = 0;
-	int32_t callcnt = 0;
-  	uint64_t tmp_list = 0;
-
-        if (!(loc && loc->inode)) {                     
-                STACK_UNWIND (frame, -1, EINVAL, NULL, NULL);
-                return 0;
-        }                                                    
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	loc_copy (&local->loc1, loc);
-
-	if (IA_ISDIR (loc->inode->ia_type)) {
-		local->call_count = 1;
-      
-                STACK_WIND (frame,
-                            unify_setattr_cbk,
-                            NS (this),
-                            NS (this)->fops->setattr,
-                            loc, stbuf, valid);
-        } else {
-		inode_ctx_get (loc->inode, this, &tmp_list);
-		local->list = (int16_t *)(long)tmp_list;
-
-		for (index = 0; local->list[index] != -1; index++) {
-			local->call_count++;
-			callcnt++;
-		}
-      
-		for (index = 0; local->list[index] != -1; index++) {
-			STACK_WIND (frame,
-				    unify_setattr_cbk,
-				    priv->xl_array[local->list[index]],
-				    priv->xl_array[local->list[index]]->fops->setattr,
-				    loc, stbuf, valid);
-
-			if (!--callcnt)
-				break;
-		}
-	}
-
-	return 0;
-}
-
-
-int32_t
-unify_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                struct iatt *stbuf, int32_t valid)
-{
-	unify_local_t *local = NULL;
-	xlator_t *child = NULL;
-	uint64_t tmp_child = 0;
-
-	UNIFY_CHECK_FD_AND_UNWIND_ON_ERR(fd);
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-
-	if (!fd_ctx_get (fd, this, &tmp_child)) {
-		/* If its set, then its file */
-		child = (xlator_t *)(long)tmp_child;		     
-
-		local->call_count = 2;
-
-		STACK_WIND (frame, unify_setattr_cbk, child,
-			    child->fops->fsetattr, fd, stbuf, valid);
-
-		STACK_WIND (frame, unify_setattr_cbk, NS(this),
-			    NS(this)->fops->fsetattr, fd, stbuf, valid);
-	} else {
-		local->call_count = 1;
-    
-		STACK_WIND (frame, unify_setattr_cbk,
-			    NS(this), NS(this)->fops->fsetattr,
-			    fd, stbuf, valid);
-	}
-  
-	return 0;
-}
-
-
-/**
- * unify_truncate_cbk - 
- */
-int32_t
-unify_truncate_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    struct iatt *prebuf,
-                    struct iatt *postbuf)
-{
-	int32_t callcnt = 0;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-	call_frame_t *prev_frame = cookie;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-    
-		if (op_ret == -1) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"child(%s): path(%s): %s", 
-				prev_frame->this->name, 
-				(local->loc1.path)?local->loc1.path:"", 
-				strerror (op_errno));
-			local->op_errno = op_errno;
-			if (!((op_errno == ENOENT) && priv->optimist))
-				local->op_ret = -1;
-		}
-
-		if (op_ret >= 0) {
-			if (NS (this) == prev_frame->this) {
-				local->ia_ino = postbuf->ia_ino;
-				/* If the entry is directory, get the 
-				   stat from NS node */
-				if (IA_ISDIR (postbuf->ia_type) ||
-				    !local->stbuf.ia_blksize) {
-					local->stbuf = *prebuf;
-                                        local->poststbuf = *postbuf;
-				}
-			}
-
-			if ((!IA_ISDIR (postbuf->ia_type)) &&
-			    (NS (this) != prev_frame->this)) {
-				/* If file, take the stat info from 
-				   Storage node. */
-				local->stbuf = *prebuf;
-                                local->poststbuf = *postbuf;
-			}
-		}
-	}
-	UNLOCK (&frame->lock);
-    
-	if (!callcnt) {
-		if (local->ia_ino) {
-			local->stbuf.ia_ino = local->ia_ino;
-                        local->poststbuf.ia_ino = local->ia_ino;
-                } else {
-			local->op_ret = -1;
-                }
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, local->op_ret, local->op_errno, 
-			      &local->stbuf, &local->poststbuf);
-	}
-
-	return 0;
-}
-
-
-/**
- * unify_truncate - 
- */
-int32_t
-unify_truncate (call_frame_t *frame,
-		xlator_t *this,
-		loc_t *loc,
-		off_t offset)
-{
-	unify_local_t *local = NULL;
-	unify_private_t *priv = this->private;
-	int32_t index = 0;
-	int32_t callcnt = 0;
-  	uint64_t tmp_list = 0;
-
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	loc_copy (&local->loc1, loc);
-	local->ia_ino = loc->inode->ino;
-
-	if (IA_ISDIR (loc->inode->ia_type)) {
-		local->call_count = 1;
-      
-		STACK_WIND (frame,
-			    unify_truncate_cbk,
-			    NS(this),
-			    NS(this)->fops->truncate,
-			    loc,
-                            0);
-	} else {
-		local->op_ret = 0;
-		inode_ctx_get (loc->inode, this, &tmp_list);
-		local->list = (int16_t *)(long)tmp_list;
-
-		for (index = 0; local->list[index] != -1; index++) {
-			local->call_count++;
-			callcnt++;
-		}
-      
-		/* Don't send offset to NS truncate */
-		STACK_WIND (frame, unify_truncate_cbk, NS(this),
-			    NS(this)->fops->truncate, loc, 0);
-		callcnt--;
-
-		for (index = 0; local->list[index] != -1; index++) {
-			if (NS(this) != priv->xl_array[local->list[index]]) {
-				STACK_WIND (frame,
-					    unify_truncate_cbk,
-					    priv->xl_array[local->list[index]],
-					    priv->xl_array[local->list[index]]->fops->truncate,
-					    loc,
-					    offset);
-				if (!--callcnt)
-					break;
-			}
-		}
-	}
-
-	return 0;
-}
-
-/**
- * unify_readlink_cbk - 
- */
-int32_t
-unify_readlink_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    const char *path,
-                    struct iatt *sbuf)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, path, sbuf);
-	return 0;
-}
-
-/**
- * unify_readlink - Read the link only from the storage node.
- */
-int32_t
-unify_readlink (call_frame_t *frame,
-		xlator_t *this,
-		loc_t *loc,
-		size_t size)
-{
-	unify_private_t *priv = this->private;
-	int32_t entry_count = 0;
-	int16_t *list = NULL;
-	int16_t index = 0;
-  	uint64_t tmp_list = 0;
-  
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-	
-	inode_ctx_get (loc->inode, this, &tmp_list);
-	list = (int16_t *)(long)tmp_list;
-
-	for (index = 0; list[index] != -1; index++)
-		entry_count++;
-
-	if (entry_count >= 2) {
-		for (index = 0; list[index] != -1; index++) {
-			if (priv->xl_array[list[index]] != NS(this)) {
-				STACK_WIND (frame,
-					    unify_readlink_cbk,
-					    priv->xl_array[list[index]],
-					    priv->xl_array[list[index]]->fops->readlink,
-					    loc,
-					    size);
-				break;
-			}
-		}
-	} else {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"returning ENOENT, no softlink files found "
-			"on storage node");
-		STACK_UNWIND (frame, -1, ENOENT, NULL);
-	}
-
-	return 0;
-}
-
-
-/**
- * unify_unlink_cbk - 
- */
-int32_t
-unify_unlink_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-                  struct iatt *preparent,
-                  struct iatt *postparent)
-{
-	int32_t callcnt = 0;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-		if (op_ret == 0  || ((op_errno == ENOENT) && priv->optimist))
-			local->op_ret = 0;
-		if (op_ret == -1)
-			local->op_errno = op_errno;
-
-                if (((call_frame_t *)cookie)->this == NS(this)) {
-                        local->oldpreparent = *preparent;
-                        local->oldpostparent = *postparent;
-                }
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, local->op_ret, local->op_errno,
-                              &local->oldpreparent, &local->oldpostparent);
-	}
-
-	return 0;
-}
-
-
-/**
- * unify_unlink - 
- */
-int32_t
-unify_unlink (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc)
-{
-	unify_private_t *priv = this->private;
-	unify_local_t *local = NULL;
-	int16_t *list = NULL;
-	int16_t index = 0;
-  	uint64_t tmp_list = 0;
-
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	loc_copy (&local->loc1, loc);
-
-	inode_ctx_get (loc->inode, this, &tmp_list);
-	list = (int16_t *)(long)tmp_list;
-
-	for (index = 0; list[index] != -1; index++)
-		local->call_count++;
-
-	if (local->call_count) {
-		for (index = 0; list[index] != -1; index++) {
-			char need_break = (list[index+1] == -1);
-			STACK_WIND (frame,
-				    unify_unlink_cbk,
-				    priv->xl_array[list[index]],
-				    priv->xl_array[list[index]]->fops->unlink,
-				    loc);
-			if (need_break)
-				break;
-		}
-	} else {
-		gf_log (this->name, GF_LOG_ERROR,
-			"%s: returning ENOENT", loc->path);
-		STACK_UNWIND (frame, -1, ENOENT, NULL, NULL);
-	}
-
-	return 0;
-}
-
-
-/**
- * unify_readv_cbk - 
- */
-int32_t
-unify_readv_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 struct iovec *vector,
-		 int32_t count,
-		 struct iatt *stbuf,
-                 struct iobref *iobref)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, vector, count, stbuf, iobref);
-	return 0;
-}
-
-/**
- * unify_readv - 
- */
-int32_t
-unify_readv (call_frame_t *frame,
-	     xlator_t *this,
-	     fd_t *fd,
-	     size_t size,
-	     off_t offset)
-{
-	UNIFY_CHECK_FD_CTX_AND_UNWIND_ON_ERR (fd);
-	xlator_t *child = NULL;
-	uint64_t tmp_child = 0;
-
-	fd_ctx_get (fd, this, &tmp_child);
-	child = (xlator_t *)(long)tmp_child;		     
-
-	STACK_WIND (frame,
-		    unify_readv_cbk,
-		    child,
-		    child->fops->readv,
-		    fd,
-		    size,
-		    offset);
-
-
-	return 0;
-}
-
-/**
- * unify_writev_cbk - 
- */
-int32_t
-unify_writev_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-                  struct iatt *prebuf,
-		  struct iatt *postbuf)
-{
-	unify_local_t *local = NULL;
-
-        local = frame->local;
-
-        local->stbuf = *prebuf;
-        local->stbuf.ia_ino = local->ia_ino;
-
-        local->poststbuf = *postbuf;
-        local->poststbuf.ia_ino = local->ia_ino;
-
-	STACK_UNWIND (frame, op_ret, op_errno,
-                      &local->stbuf, &local->poststbuf);
-	return 0;
-}
-
-/**
- * unify_writev - 
- */
-int32_t
-unify_writev (call_frame_t *frame,
-	      xlator_t *this,
-	      fd_t *fd,
-	      struct iovec *vector,
-	      int32_t count,
-	      off_t off,
-              struct iobref *iobref)
-{
-	UNIFY_CHECK_FD_CTX_AND_UNWIND_ON_ERR (fd);
-	xlator_t *child = NULL;
-	uint64_t tmp_child = 0;
-	unify_local_t *local = NULL;
-
-	INIT_LOCAL (frame, local);
-        local->ia_ino = fd->inode->ino;
-
-	fd_ctx_get (fd, this, &tmp_child);
-	child = (xlator_t *)(long)tmp_child;		     
-
-	STACK_WIND (frame,
-		    unify_writev_cbk,
-		    child,
-		    child->fops->writev,
-		    fd,
-		    vector,
-		    count,
-		    off,
-                    iobref);
-
-	return 0;
-}
-
-/**
- * unify_ftruncate -
- */
-int32_t
-unify_ftruncate (call_frame_t *frame,
-		 xlator_t *this,
-		 fd_t *fd,
-		 off_t offset)
-{
-	xlator_t *child = NULL;
-	unify_local_t *local = NULL;
-	uint64_t tmp_child = 0;
-
-	UNIFY_CHECK_FD_CTX_AND_UNWIND_ON_ERR(fd);
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	local->op_ret = 0;
-
-	fd_ctx_get (fd, this, &tmp_child);
-	child = (xlator_t *)(long)tmp_child;		     
-
-	local->call_count = 2;
-  
-	STACK_WIND (frame, unify_truncate_cbk,
-		    child, child->fops->ftruncate,
-		    fd, offset);
-  
-	STACK_WIND (frame, unify_truncate_cbk,
-		    NS(this), NS(this)->fops->ftruncate,
-		    fd, 0);
-  
-	return 0;
-}
-
-
-/**
- * unify_flush_cbk - 
- */
-int32_t
-unify_flush_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno)
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-/**
- * unify_flush -
- */
-int32_t
-unify_flush (call_frame_t *frame,
-	     xlator_t *this,
-	     fd_t *fd)
-{
-	UNIFY_CHECK_FD_CTX_AND_UNWIND_ON_ERR (fd);
-	xlator_t *child = NULL;
-	uint64_t tmp_child = 0;
-
-	fd_ctx_get (fd, this, &tmp_child);
-	child = (xlator_t *)(long)tmp_child;		     
-
-	STACK_WIND (frame, unify_flush_cbk, child, 
-		    child->fops->flush, fd);
-
-	return 0;
-}
-
-
-/**
- * unify_fsync_cbk - 
- */
-int32_t
-unify_fsync_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-                 struct iatt *prebuf,
-                 struct iatt *postbuf)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, prebuf, postbuf);
-	return 0;
-}
-
-/**
- * unify_fsync - 
- */
-int32_t
-unify_fsync (call_frame_t *frame,
-	     xlator_t *this,
-	     fd_t *fd,
-	     int32_t flags)
-{
-	UNIFY_CHECK_FD_CTX_AND_UNWIND_ON_ERR (fd);
-	xlator_t *child = NULL;
-	uint64_t tmp_child = 0;
-
-	fd_ctx_get (fd, this, &tmp_child);
-	child = (xlator_t *)(long)tmp_child;		     
-
-	STACK_WIND (frame, unify_fsync_cbk, child,
-		    child->fops->fsync, fd, flags);
-
-	return 0;
-}
-
-/**
- * unify_fstat - Send fstat FOP to Namespace only if its directory, and to 
- *     both namespace and the storage node if its a file.
- */
-int32_t
-unify_fstat (call_frame_t *frame,
-	     xlator_t *this,
-	     fd_t *fd)
-{
-	unify_local_t *local = NULL;
-	xlator_t *child = NULL;
-	uint64_t tmp_child = 0;
-
-	UNIFY_CHECK_FD_AND_UNWIND_ON_ERR(fd);
-
-	INIT_LOCAL (frame, local);
-	local->ia_ino = fd->inode->ino;
-
-	if (!fd_ctx_get (fd, this, &tmp_child)) {
-		/* If its set, then its file */
-		child = (xlator_t *)(long)tmp_child;		     
-		local->call_count = 2;
-
-		STACK_WIND (frame, unify_buf_cbk, child,
-			    child->fops->fstat, fd);
-
-		STACK_WIND (frame, unify_buf_cbk, NS(this),
-			    NS(this)->fops->fstat, fd);
-
-	} else {
-		/* this is an directory */
-		local->call_count = 1;
-		STACK_WIND (frame, unify_buf_cbk, NS(this),
-			    NS(this)->fops->fstat, fd);
-	}
-
-	return 0;
-}
-
-/**
- * unify_getdents_cbk - 
- */
-int32_t
-unify_getdents_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    dir_entry_t *entry,
-		    int32_t count)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, entry, count);
-	return 0;
-}
-
-/**
- * unify_getdents - send the FOP request to all the nodes.
- */
-int32_t
-unify_getdents (call_frame_t *frame,
-		xlator_t *this,
-		fd_t *fd,
-		size_t size,
-		off_t offset,
-		int32_t flag)
-{
-	UNIFY_CHECK_FD_AND_UNWIND_ON_ERR (fd);
-
-	STACK_WIND (frame, unify_getdents_cbk, NS(this),
-		    NS(this)->fops->getdents, fd, size, offset, flag);
-
-	return 0;
-}
-
-
-/**
- * unify_readdir_cbk - 
- */
-int32_t
-unify_readdir_cbk (call_frame_t *frame,
-		   void *cookie,
-		   xlator_t *this,
-		   int32_t op_ret,
-		   int32_t op_errno,
-		   gf_dirent_t *buf)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, buf);
-
-	return 0;
-}
-
-/**
- * unify_readdir - send the FOP request to all the nodes.
- */
-int32_t
-unify_readdir (call_frame_t *frame,
-	       xlator_t *this,
-	       fd_t *fd,
-	       size_t size,
-	       off_t offset)
-{
-	UNIFY_CHECK_FD_AND_UNWIND_ON_ERR (fd);
-
-	STACK_WIND (frame, unify_readdir_cbk, NS(this),
-		    NS(this)->fops->readdir, fd, size, offset);
-
-	return 0;
-}
-
-
-int32_t
-unify_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, gf_dirent_t *buf)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, buf);
-
-	return 0;
-}
-
-
-int32_t
-unify_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-                off_t offset)
-{
-	UNIFY_CHECK_FD_AND_UNWIND_ON_ERR (fd);
-
-	STACK_WIND (frame, unify_readdirp_cbk, NS(this),
-		    NS(this)->fops->readdirp, fd, size, offset);
-
-	return 0;
-}
-
-
-/**
- * unify_fsyncdir_cbk - 
- */
-int32_t
-unify_fsyncdir_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno)
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-
-	return 0;
-}
-
-/**
- * unify_fsyncdir -
- */
-int32_t
-unify_fsyncdir (call_frame_t *frame,
-		xlator_t *this,
-		fd_t *fd,
-		int32_t flags)
-{
-	UNIFY_CHECK_FD_AND_UNWIND_ON_ERR (fd);
-
-	STACK_WIND (frame, unify_fsyncdir_cbk,
-		    NS(this), NS(this)->fops->fsyncdir, fd, flags);
-
-	return 0;
-}
-
-/**
- * unify_lk_cbk - UNWIND frame with the proper return arguments.
- */
-int32_t
-unify_lk_cbk (call_frame_t *frame,
-	      void *cookie,
-	      xlator_t *this,
-	      int32_t op_ret,
-	      int32_t op_errno,
-	      struct gf_flock *lock)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, lock);
-	return 0;
-}
-
-/**
- * unify_lk - Send it to all the storage nodes, (should be 1) which has file.
- */
-int32_t
-unify_lk (call_frame_t *frame,
-	  xlator_t *this,
-	  fd_t *fd,
-	  int32_t cmd,
-	  struct gf_flock *lock)
-{
-	UNIFY_CHECK_FD_CTX_AND_UNWIND_ON_ERR (fd);
-	xlator_t *child = NULL;
-	uint64_t tmp_child = 0;
-
-	fd_ctx_get (fd, this, &tmp_child);
-	child = (xlator_t *)(long)tmp_child;		     
-
-	STACK_WIND (frame, unify_lk_cbk, child,
-		    child->fops->lk, fd, cmd, lock);
-
-	return 0;
-}
-
-
-int32_t
-unify_setxattr_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno);
-
-static int32_t
-unify_setxattr_file_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno)
-{
-	unify_private_t *private = this->private;
-	unify_local_t *local = frame->local;
-	xlator_t *sched_xl = NULL;
-	struct sched_ops *sched_ops = NULL;
-
-	if (op_ret == -1) {
-		if (!ENOTSUP)
-			gf_log (this->name, GF_LOG_ERROR,
-				"setxattr with XATTR_CREATE on ns: "
-				"path(%s) key(%s): %s",
-				local->loc1.path, local->name, 
-				strerror (op_errno));
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, op_ret, op_errno);
-		return 0;
-	} 
-
-	LOCK (&frame->lock);
-	{
-		local->failed = 0;
-		local->op_ret = 0;
-		local->op_errno = 0;
-		local->call_count = 1;
-	}
-	UNLOCK (&frame->lock);
-
-	/* schedule XATTR_CREATE on one of the child node */
-	sched_ops = private->sched_ops;
-    
-	/* Send create request to the scheduled node now */
-	sched_xl = sched_ops->schedule (this, local->name); 
-	if (!sched_xl) {
-		STACK_UNWIND (frame, -1, ENOTCONN);
-		return 0;
-	}
-
-	STACK_WIND (frame,
-		    unify_setxattr_cbk,
-		    sched_xl,
-		    sched_xl->fops->setxattr,
-		    &local->loc1,
-		    local->dict,
-		    local->flags);
-	return 0;
-}
-
-/**
- * unify_setxattr_cbk - When all the child nodes return, UNWIND frame.
- */
-int32_t
-unify_setxattr_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno)
-{
-	int32_t callcnt = 0;
-	unify_local_t *local = frame->local;
-	call_frame_t *prev_frame = cookie;
-	dict_t *dict = NULL;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-    
-		if (op_ret == -1) {
-			gf_log (this->name, (((op_errno == ENOENT) || 
-					      (op_errno == ENOTSUP))? 
-					     GF_LOG_DEBUG : GF_LOG_ERROR), 
-				"child(%s): path(%s): %s", 
-				prev_frame->this->name, 
-				(local->loc1.path)?local->loc1.path:"", 
-				strerror (op_errno));
-			if (local->failed == -1) {
-				local->failed = 1;
-			}
-			local->op_errno = op_errno;
-		} else {
-			local->failed = 0;
-			local->op_ret = op_ret;
-		}
-	}
-	UNLOCK (&frame->lock);
-  
-	if (!callcnt) {
-		if (local->failed && local->name && 
-		    ZR_FILE_CONTENT_REQUEST(local->name)) {      
-			dict = get_new_dict ();
-			dict_set (dict, local->dict->members_list->key, 
-				  data_from_dynptr(NULL, 0));
-			dict_ref (dict);
-
-			local->call_count = 1;
-
-			STACK_WIND (frame,
-				    unify_setxattr_file_cbk,
-				    NS(this),
-				    NS(this)->fops->setxattr,
-				    &local->loc1,
-				    dict,
-				    XATTR_CREATE);
-
-			dict_unref (dict);
-			return 0;
-		}
-    
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, local->op_ret, local->op_errno);
-	}
-
-	return 0;
-}
-
-/**
- * unify_sexattr - This function should be sent to all the storage nodes, 
- *       which contains the file, (excluding namespace).
- */
-int32_t
-unify_setxattr (call_frame_t *frame,
-		xlator_t *this,
-		loc_t *loc,
-		dict_t *dict,
-		int32_t flags)
-{
-	unify_private_t *priv = this->private;
-	unify_local_t *local = NULL;
-	int16_t *list = NULL;
-	int16_t index = 0;
-	int32_t call_count = 0;
-  	uint64_t tmp_list = 0;
-	data_pair_t *trav = dict->members_list;
-
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	local->failed = -1;
-	loc_copy (&local->loc1, loc);
-
-	if (IA_ISDIR (loc->inode->ia_type)) {
-
-		if (trav && trav->key && ZR_FILE_CONTENT_REQUEST(trav->key)) {
-			/* direct the storage xlators to change file 
-			   content only if file exists */
-			local->flags = flags;
-			local->dict = dict;
-			local->name = gf_strdup (trav->key);
-			flags |= XATTR_REPLACE;
-		}
-
-		local->call_count = priv->child_count;
-		for (index = 0; index < priv->child_count; index++) {
-			STACK_WIND (frame,
-				    unify_setxattr_cbk,
-				    priv->xl_array[index],
-				    priv->xl_array[index]->fops->setxattr,
-				    loc, dict, flags);
-		}
-		return 0;
-	}
-
-	inode_ctx_get (loc->inode, this, &tmp_list);
-	list = (int16_t *)(long)tmp_list;
-
-	for (index = 0; list[index] != -1; index++) {
-		if (NS(this) != priv->xl_array[list[index]]) {
-			local->call_count++;
-			call_count++;
-		}
-	}
-  
-	if (local->call_count) {
-		for (index = 0; list[index] != -1; index++) {
-			if (priv->xl_array[list[index]] != NS(this)) {
-				STACK_WIND (frame,
-					    unify_setxattr_cbk,
-					    priv->xl_array[list[index]],
-					    priv->xl_array[list[index]]->fops->setxattr,
-					    loc,
-					    dict,
-					    flags);
-				if (!--call_count)
-					break;
-			}
-		}
-		return 0;
-	}
-
-	/* No entry in storage nodes */
-	gf_log (this->name, GF_LOG_DEBUG, 
-		"returning ENOENT, file not found on storage node.");
-	STACK_UNWIND (frame, -1, ENOENT);
-
-	return 0;
-}
-
-
-/**
- * unify_getxattr_cbk - This function is called from only one child, so, no
- *     need of any lock or anything else, just send it to above layer 
- */
-int32_t
-unify_getxattr_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    dict_t *value)
-{
-	int32_t callcnt = 0;
-	dict_t *local_value = NULL;
-	unify_local_t *local = frame->local;
-	call_frame_t *prev_frame = cookie;
-  
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-    
-		if (op_ret == -1) {
-			local->op_errno = op_errno;
-			gf_log (this->name, 
-				(((op_errno == ENOENT) || 
-				  (op_errno == ENODATA) || 
-				  (op_errno == ENOTSUP)) ? 
-				 GF_LOG_DEBUG : GF_LOG_ERROR), 
-				"child(%s): path(%s): %s", 
-				prev_frame->this->name, 
-				(local->loc1.path)?local->loc1.path:"", 
-				strerror (op_errno));
-		} else {
-			if (!local->dict)
-				local->dict = dict_ref (value);
-			local->op_ret = op_ret;
-		}
-	}
-	UNLOCK (&frame->lock);
-  
-	if (!callcnt) {
-		local_value = local->dict;
-		local->dict = NULL;
-      
-		STACK_UNWIND (frame, local->op_ret, local->op_errno, 
-			      local_value);
-      
-		if (local_value)
-			dict_unref (local_value);
-	} 
-
-	return 0;
-}
-
-
-/** 
- * unify_getxattr - This FOP is sent to only the storage node.
- */
-int32_t
-unify_getxattr (call_frame_t *frame,
-		xlator_t *this,
-		loc_t *loc,
-		const char *name)
-{
-	unify_private_t *priv = this->private;
-	int16_t *list = NULL;
-	int16_t index = 0;
-	int16_t count = 0;
-	unify_local_t *local = NULL;
-  	uint64_t tmp_list = 0;
-
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-	INIT_LOCAL (frame, local);
-
-	if (IA_ISDIR (loc->inode->ia_type)) {
-		local->call_count = priv->child_count;
-		for (index = 0; index < priv->child_count; index++)
-			STACK_WIND (frame,
-				    unify_getxattr_cbk,
-				    priv->xl_array[index],
-				    priv->xl_array[index]->fops->getxattr,
-				    loc,
-				    name);
-		return 0;
-	}
-
-	inode_ctx_get (loc->inode, this, &tmp_list);
-	list = (int16_t *)(long)tmp_list;
-
-	for (index = 0; list[index] != -1; index++) {
-		if (NS(this) != priv->xl_array[list[index]]) {
-			local->call_count++;
-			count++;
-		}
-	}
-
-	if (count) {
-		for (index = 0; list[index] != -1; index++) {
-			if (priv->xl_array[list[index]] != NS(this)) {
-				STACK_WIND (frame,
-					    unify_getxattr_cbk,
-					    priv->xl_array[list[index]],
-					    priv->xl_array[list[index]]->fops->getxattr,
-					    loc,
-					    name);
-				if (!--count)
-					break;
-			}
-		}
-	} else {
-		dict_t *tmp_dict = get_new_dict ();
-		gf_log (this->name, GF_LOG_DEBUG, 
-			"%s: returning ENODATA, no file found on storage node",
-			loc->path);
-		STACK_UNWIND (frame, -1, ENODATA, tmp_dict);
-		dict_destroy (tmp_dict);
-	}
-
-	return 0;
-}
-
-/**
- * unify_removexattr_cbk - Wait till all the child node returns the call
- *      and then UNWIND to above layer.
- */
-int32_t
-unify_removexattr_cbk (call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno)
-{
-	int32_t callcnt = 0;
-	unify_local_t *local = frame->local;
-	call_frame_t *prev_frame = cookie;
-
-	LOCK (&frame->lock);
-	{ 
-		callcnt = --local->call_count;
-		if (op_ret == -1) {
-			local->op_errno = op_errno;
-			if (op_errno != ENOTSUP)
-				gf_log (this->name, GF_LOG_ERROR, 
-					"child(%s): path(%s): %s", 
-					prev_frame->this->name, 
-					local->loc1.path, strerror (op_errno));
-		} else {
-			local->op_ret = op_ret;
-		}
-	}
-	UNLOCK (&frame->lock);  
-
-	if (!callcnt) {
-		STACK_UNWIND (frame, local->op_ret, local->op_errno);
-	}
-
-	return 0;
-}
-
-/**
- * unify_removexattr - Send it to all the child nodes which has the files.
- */
-int32_t
-unify_removexattr (call_frame_t *frame,
-		   xlator_t *this,
-		   loc_t *loc,
-		   const char *name)
-{
-	unify_private_t *priv = this->private;
-	unify_local_t *local = NULL;
-	int16_t *list = NULL;
-	int16_t index = 0;
-	int32_t call_count = 0;
-  	uint64_t tmp_list = 0;
-
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-
-	if (IA_ISDIR (loc->inode->ia_type)) {
-		local->call_count = priv->child_count;
-		for (index = 0; index < priv->child_count; index++)
-			STACK_WIND (frame, 		    
-				    unify_removexattr_cbk,
-				    priv->xl_array[index],
-				    priv->xl_array[index]->fops->removexattr,
-				    loc,
-				    name);
-
-		return 0;
-	}
-
-	inode_ctx_get (loc->inode, this, &tmp_list);
-	list = (int16_t *)(long)tmp_list;
-
-	for (index = 0; list[index] != -1; index++) {
-		if (NS(this) != priv->xl_array[list[index]]) {
-			local->call_count++;
-			call_count++;
-		}
-	}
-
-	if (local->call_count) {
-		for (index = 0; list[index] != -1; index++) {
-			if (priv->xl_array[list[index]] != NS(this)) {
-				STACK_WIND (frame,
-					    unify_removexattr_cbk,
-					    priv->xl_array[list[index]],
-					    priv->xl_array[list[index]]->fops->removexattr,
-					    loc,
-					    name);
-				if (!--call_count)
-					break;
-			}
-		}
-		return 0;
-	} 
-
-	gf_log (this->name, GF_LOG_DEBUG, 
-		"%s: returning ENOENT, not found on storage node.", loc->path);
-	STACK_UNWIND (frame, -1, ENOENT);
-
-	return 0;
-}
-
-
-int32_t 
-unify_mknod_unlink_cbk (call_frame_t *frame,
-			void *cookie,
-			xlator_t *this,
-			int32_t op_ret,
-			int32_t op_errno,
-                        struct iatt *preparent,
-                        struct iatt *postparent)
-{
-	unify_local_t *local = frame->local;
-
-	if (op_ret == -1)
-		gf_log (this->name, GF_LOG_ERROR, 
-			"%s: %s", local->loc1.path, strerror (op_errno));
-  
-	unify_local_wipe (local);
-	/* No log required here as this -1 is for mknod call */
-	STACK_UNWIND (frame, -1, local->op_errno, NULL, NULL);
-	return 0;
-}
-
-/**
- * unify_mknod_cbk - 
- */
-int32_t
-unify_mknod_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 inode_t *inode,
-                 struct iatt *buf,
-                 struct iatt *preparent,
-                 struct iatt *postparent)
-{
-	unify_local_t *local = frame->local;
-
-	if (op_ret == -1) {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"mknod failed on storage node, sending unlink to "
-			"namespace");
-		local->op_errno = op_errno;
-		STACK_WIND (frame,
-			    unify_mknod_unlink_cbk,
-			    NS(this),
-			    NS(this)->fops->unlink,
-			    &local->loc1);
-		return 0;
-	}
-  
-	local->stbuf = *buf;
-	local->stbuf.ia_ino = local->ia_ino;
-	unify_local_wipe (local);
-	STACK_UNWIND (frame, op_ret, op_errno, inode, &local->stbuf,
-                      &local->oldpreparent, &local->oldpostparent);
-	return 0;
-}
-
-/**
- * unify_ns_mknod_cbk - 
- */
-int32_t
-unify_ns_mknod_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    inode_t *inode,
-                    struct iatt *buf,
-                    struct iatt *preparent,
-                    struct iatt *postparent)
-{
-	struct sched_ops *sched_ops = NULL;
-	xlator_t *sched_xl = NULL;
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	int16_t *list = NULL;
-	int16_t index = 0;
-	call_frame_t *prev_frame = cookie;
-
-	if (op_ret == -1) {
-		/* No need to send mknod request to other servers, 
-		 * as namespace action failed 
-		 */
-		gf_log (this->name, GF_LOG_ERROR, 
-			"child(%s): path(%s): %s", 
-			prev_frame->this->name, local->loc1.path, 
-			strerror (op_errno));
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, op_ret, op_errno, inode, buf,
-                              preparent, postparent);
-		return 0;
-	}
-  
-	/* Create one inode for this entry */
-	local->op_ret = 0;
-	local->stbuf = *buf;
-	local->ia_ino = buf->ia_ino;
-
-        local->oldpreparent = *preparent;
-        local->oldpostparent = *postparent;
-
-	list = GF_CALLOC (1, sizeof (int16_t) * 3, gf_unify_mt_int16_t);
-	ERR_ABORT (list);
-	list[0] = priv->child_count;
-	list[2] = -1;
-	inode_ctx_put (inode, this, (uint64_t)(long)list);
-
-	sched_ops = priv->sched_ops;
-
-	/* Send mknod request to scheduled node now */
-	sched_xl = sched_ops->schedule (this, local->loc1.path); 
-	if (!sched_xl) {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"mknod failed on storage node, no node online "
-			"at the moment, sending unlink to NS");
-		local->op_errno = ENOTCONN;
-		STACK_WIND (frame,
-			    unify_mknod_unlink_cbk,
-			    NS(this),
-			    NS(this)->fops->unlink,
-			    &local->loc1);
-      
-		return 0;
-	}
-
-	for (index = 0; index < priv->child_count; index++)
-		if (sched_xl == priv->xl_array[index])
-			break;
-	list[1] = index;
-  
-	STACK_WIND (frame,  unify_mknod_cbk,
-		    sched_xl,  sched_xl->fops->mknod,
-		    &local->loc1, local->mode, local->dev);
-
-	return 0;
-}
-
-/**
- * unify_mknod - Create a device on namespace first, and later create on 
- *       the storage node.
- */
-int32_t
-unify_mknod (call_frame_t *frame,
-	     xlator_t *this,
-	     loc_t *loc,
-	     mode_t mode,
-	     dev_t rdev)
-{
-	unify_local_t *local = NULL;
-  
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	local->mode = mode;
-	local->dev = rdev;
-	loc_copy (&local->loc1, loc);
-	if (local->loc1.path == NULL) {
-		gf_log (this->name, GF_LOG_CRITICAL, "Not enough memory :O");
-		STACK_UNWIND (frame, -1, ENOMEM, loc->inode, NULL);
-		return 0;
-	}
-
-	STACK_WIND (frame,
-		    unify_ns_mknod_cbk,
-		    NS(this),
-		    NS(this)->fops->mknod,
-		    loc,
-		    mode,
-		    rdev);
-
-	return 0;
-}
-
-int32_t 
-unify_symlink_unlink_cbk (call_frame_t *frame,
-			  void *cookie,
-			  xlator_t *this,
-			  int32_t op_ret,
-			  int32_t op_errno,
-                          struct iatt *preparent,
-                          struct iatt *postparent)
-{
-	unify_local_t *local = frame->local;
-	if (op_ret == -1)
-		gf_log (this->name, GF_LOG_ERROR, 
-			"%s: %s", local->loc1.path, strerror (op_errno));
-
-	unify_local_wipe (local);
-	STACK_UNWIND (frame, -1, local->op_errno, NULL, NULL);
-	return 0;
-}
-
-/**
- * unify_symlink_cbk - 
- */
-int32_t
-unify_symlink_cbk (call_frame_t *frame,
-		   void *cookie,
-		   xlator_t *this,
-		   int32_t op_ret,
-		   int32_t op_errno,
-		   inode_t *inode,
-                   struct iatt *buf,
-                   struct iatt *preparent,
-                   struct iatt *postparent)
-{
-	unify_local_t *local = frame->local;
-
-	if (op_ret == -1) {
-		/* Symlink on storage node failed, hence send unlink 
-		   to the NS node */
-		local->op_errno = op_errno;
-		gf_log (this->name, GF_LOG_ERROR, 
-			"symlink on storage node failed, sending unlink "
-			"to namespace");
-
-		STACK_WIND (frame,
-			    unify_symlink_unlink_cbk,
-			    NS(this),
-			    NS(this)->fops->unlink,
-			    &local->loc1);
-    
-		return 0;
-	}
-  
-	local->stbuf = *buf;
-	local->stbuf.ia_ino = local->ia_ino;
-	unify_local_wipe (local);
-	STACK_UNWIND (frame, op_ret, op_errno, inode, &local->stbuf,
-                      &local->oldpreparent, &local->oldpostparent);
-
-	return 0;
-}
-
-/**
- * unify_ns_symlink_cbk - 
- */
-int32_t
-unify_ns_symlink_cbk (call_frame_t *frame,
-		      void *cookie,
-		      xlator_t *this,
-		      int32_t op_ret,
-		      int32_t op_errno,
-		      inode_t *inode,
-                      struct iatt *buf,
-                      struct iatt *preparent,
-                      struct iatt *postparent)
-{
-
-	struct sched_ops *sched_ops = NULL;
-	xlator_t *sched_xl = NULL;
-	int16_t *list = NULL;
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	int16_t index = 0;
-
-	if (op_ret == -1) {
-		/* No need to send symlink request to other servers, 
-		 * as namespace action failed 
-		 */
-		gf_log (this->name, GF_LOG_ERROR, 
-			"namespace: path(%s): %s", 
-			local->loc1.path, strerror (op_errno));
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, op_ret, op_errno, NULL, buf,
-                              preparent, postparent);
-		return 0;
-	}
-  
-	/* Create one inode for this entry */
-	local->op_ret = 0;
-	local->ia_ino = buf->ia_ino;
-  
-        local->oldpreparent = *preparent;
-        local->oldpostparent = *postparent;
-
-	/* Start the mapping list */
-
-	list = GF_CALLOC (1, sizeof (int16_t) * 3, gf_unify_mt_int16_t);
-	ERR_ABORT (list);
-	list[0] = priv->child_count; //namespace's index
-	list[2] = -1;
-	inode_ctx_put (inode, this, (uint64_t)(long)list);
-
-	sched_ops = priv->sched_ops;
-
-	/* Send symlink request to all the nodes now */
-	sched_xl = sched_ops->schedule (this, local->loc1.path); 
-	if (!sched_xl) {
-		/* Symlink on storage node failed, hence send unlink 
-		   to the NS node */
-		local->op_errno = ENOTCONN;
-		gf_log (this->name, GF_LOG_ERROR, 
-			"symlink on storage node failed, no node online, "
-			"sending unlink to namespace");
-      
-		STACK_WIND (frame,
-			    unify_symlink_unlink_cbk,
-			    NS(this),
-			    NS(this)->fops->unlink,
-			    &local->loc1);
-      
-		return 0;
-	}
-
-	for (index = 0; index < priv->child_count; index++)
-		if (sched_xl == priv->xl_array[index])
-			break;
-	list[1] = index;
-	
-	STACK_WIND (frame,
-		    unify_symlink_cbk,
-		    sched_xl,
-		    sched_xl->fops->symlink,
-		    local->name,
-		    &local->loc1);
-
-	return 0;
-}
-
-/**
- * unify_symlink - 
- */
-int32_t
-unify_symlink (call_frame_t *frame,
-	       xlator_t *this,
-	       const char *linkpath,
-	       loc_t *loc)
-{
-	unify_local_t *local = NULL;
-  
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	loc_copy (&local->loc1, loc);
-	local->name = gf_strdup (linkpath);
-
-	if ((local->name == NULL) || 
-	    (local->loc1.path == NULL)) {
-		gf_log (this->name, GF_LOG_CRITICAL, "Not enough memory :O");
-		STACK_UNWIND (frame, -1, ENOMEM, loc->inode, NULL);
-		return 0;
-	}
-
-	STACK_WIND (frame,
-		    unify_ns_symlink_cbk,
-		    NS(this),
-		    NS(this)->fops->symlink,
-		    linkpath,
-		    loc);
-
-	return 0;
-}
-
-
-int32_t 
-unify_rename_unlink_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno,
-                         struct iatt *preparent,
-                         struct iatt *postparent)
-{
-	int32_t callcnt = 0;
-	unify_local_t *local = frame->local;
-	call_frame_t *prev_frame = cookie;
-  
-	if (op_ret == -1) {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"child(%s): path(%s -> %s): %s", 
-			prev_frame->this->name, 
-			local->loc1.path, local->loc2.path, 
-			strerror (op_errno));
-      
-	}
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		local->stbuf.ia_ino = local->ia_ino;
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, local->op_ret, local->op_errno, 
-			      &local->stbuf);
-	}
-	return 0;
-}
-
-int32_t 
-unify_ns_rename_undo_cbk (call_frame_t *frame,
-			  void *cookie,
-			  xlator_t *this,
-			  int32_t op_ret,
-			  int32_t op_errno,
-			  struct iatt *buf,
-                          struct iatt *preoldparent,
-                          struct iatt *postoldparent,
-                          struct iatt *prenewparent,
-                          struct iatt *postnewparent)
-{
-	unify_local_t *local = frame->local;
-
-	if (op_ret == -1) {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"namespace: path(%s -> %s): %s", 
-			local->loc1.path, local->loc2.path, 
-			strerror (op_errno));
-	}
-
-	local->stbuf.ia_ino = local->ia_ino;
-	unify_local_wipe (local);
-	STACK_UNWIND (frame, local->op_ret, local->op_errno, &local->stbuf);
-	return 0;
-}
-
-int32_t 
-unify_rename_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  struct iatt *buf,
-                  struct iatt *preoldparent,
-                  struct iatt *postoldparent,
-                  struct iatt *prenewparent,
-                  struct iatt *postnewparent)
-{
-	int32_t index = 0;
-	int32_t callcnt = 0;
-	int16_t *list = NULL;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-	call_frame_t *prev_frame = cookie;
-  
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-		if (op_ret >= 0) {
-			if (!IA_ISDIR (buf->ia_type))
-				local->stbuf = *buf;
-			local->op_ret = op_ret;
-		} else {
-			gf_log (this->name, GF_LOG_ERROR, 
-				"child(%s): path(%s -> %s): %s", 
-				prev_frame->this->name, 
-				local->loc1.path, local->loc2.path, 
-				strerror (op_errno));
-			local->op_errno = op_errno;
-		}
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		local->stbuf.ia_ino = local->ia_ino;
-		if (IA_ISDIR (local->loc1.inode->ia_type)) {
-			unify_local_wipe (local);
-			STACK_UNWIND (frame, local->op_ret, local->op_errno,
-                                      &local->stbuf, &local->oldpreparent,
-                                      &local->oldpostparent, &local->newpreparent,
-                                      &local->newpostparent);
-			return 0;
-		}
-
-		if (local->op_ret == -1) {
-			/* TODO: check this logic */
-
-			/* Rename failed in storage node, successful on NS, 
-			 * hence, rename back the entries in NS */
-			/* NOTE: this will be done only if the destination 
-			 * doesn't exists, if  the destination exists, the 
-			 * job of correcting NS is left to self-heal
-			 */
-			if (!local->index) {
-				loc_t tmp_oldloc = {
-                                        /* its actual 'newloc->path' */
-					.path = local->loc2.path, 
-					.inode = local->loc1.inode,
-					.parent = local->loc2.parent
-				};
-	
-				loc_t tmp_newloc = {
-					/* Actual 'oldloc->path' */
-					.path = local->loc1.path,
-					.parent = local->loc1.parent
-				};
-
-				gf_log (this->name, GF_LOG_ERROR, 
-					"rename succussful on namespace, on "
-					"stroage node failed, reverting back");
-
-				STACK_WIND (frame,
-					    unify_ns_rename_undo_cbk,
-					    NS(this),
-					    NS(this)->fops->rename,
-					    &tmp_oldloc,
-					    &tmp_newloc);
-				return 0;
-			}
-		} else {
-			/* Rename successful on storage nodes */
-
-			int32_t idx = 0;
-			int16_t *tmp_list = NULL;
-			uint64_t tmp_list_int64 = 0;
-			if (local->loc2.inode) {
-				inode_ctx_get (local->loc2.inode, 
-					       this, &tmp_list_int64);
-				list = (int16_t *)(long)tmp_list_int64;
-
-			}
-
-			if (list) {				
-				for (index = 0; list[index] != -1; index++);
-				tmp_list = GF_CALLOC (1, index * 2, 
-                                                      gf_unify_mt_int16_t);
-				memcpy (tmp_list, list, index * 2);
-
-				for (index = 0; list[index] != -1; index++) {
-					/* TODO: Check this logic. */
-					/* If the destination file exists in 
-					 * the same storage node where we sent
-					 * 'rename' call, no need to send 
-					 * unlink 
-					 */
-					for (idx = 0; 
-					     local->list[idx] != -1; idx++) {
-						if (tmp_list[index] == local->list[idx]) {
-							tmp_list[index] = priv->child_count;
-							continue;
-						}
-					}
-	  
-					if (NS(this) != priv->xl_array[tmp_list[index]]) {
-						local->call_count++;
-						callcnt++;
-					}
-				}
-
-				if (local->call_count) {
-					if (callcnt > 1)
-						gf_log (this->name, 
-							GF_LOG_ERROR, 
-							"%s->%s: more (%d) "
-							"subvolumes have the "
-							"newloc entry", 
-							local->loc1.path, 
-							local->loc2.path, 
-							callcnt);
-
-					for (index=0; 
-					     tmp_list[index] != -1; index++) {
-						if (NS(this) != priv->xl_array[tmp_list[index]]) {		    
-							STACK_WIND (frame,
-								    unify_rename_unlink_cbk,
-								    priv->xl_array[tmp_list[index]],
-								    priv->xl_array[tmp_list[index]]->fops->unlink,
-								    &local->loc2);
-							if (!--callcnt)
-								break;
-						}
-					}
-
-					GF_FREE (tmp_list);
-					return 0;
-				}
-				if (tmp_list)
-					GF_FREE (tmp_list);
-			}
-		}
-    
-		/* Need not send 'unlink' to storage node */
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, local->op_ret, 
-			      local->op_errno, &local->stbuf,
-                              &local->oldpreparent, &local->oldpostparent,
-                              &local->newpreparent, &local->newpostparent);
-	}
-
-	return 0;
-}
-
-int32_t 
-unify_ns_rename_cbk (call_frame_t *frame,
-		     void *cookie,
-		     xlator_t *this,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     struct iatt *buf,
-                     struct iatt *preoldparent,
-                     struct iatt *postoldparent,
-                     struct iatt *prenewparent,
-                     struct iatt *postnewparent)
-{
-	int32_t index = 0;
-	int32_t callcnt = 0;
-	int16_t *list = NULL;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-
-	if (op_ret == -1) {
-		/* Free local->new_inode */
-		gf_log (this->name, GF_LOG_ERROR, 
-			"namespace: path(%s -> %s): %s", 
-			local->loc1.path, local->loc2.path, 
-			strerror (op_errno));
-
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, op_ret, op_errno, buf,
-                              preoldparent, postoldparent,
-                              prenewparent, postnewparent);
-		return 0;
-	}
-
-	local->stbuf = *buf;
-	local->ia_ino = buf->ia_ino;
-
-        local->oldpreparent = *preoldparent;
-        local->oldpostparent = *postoldparent;
-        local->newpreparent = *prenewparent;
-        local->newpostparent = *postnewparent;
-
-	/* Everything is fine. */
-	if (IA_ISDIR (buf->ia_type)) {
-		local->call_count = priv->child_count;
-		for (index=0; index < priv->child_count; index++) {
-			STACK_WIND (frame,
-				    unify_rename_cbk,
-				    priv->xl_array[index],
-				    priv->xl_array[index]->fops->rename,
-				    &local->loc1,
-				    &local->loc2);
-		}
-
-		return 0;
-	}
-
-	local->call_count = 0;  
-	/* send rename */
-	list = local->list;
-	for (index=0; list[index] != -1; index++) {
-		if (NS(this) != priv->xl_array[list[index]]) {
-			local->call_count++;
-			callcnt++;
-		}
-	}
-
-	if (local->call_count) {
-		for (index=0; list[index] != -1; index++) {
-			if (NS(this) != priv->xl_array[list[index]]) {
-				STACK_WIND (frame,
-					    unify_rename_cbk,
-					    priv->xl_array[list[index]],
-					    priv->xl_array[list[index]]->fops->rename,
-					    &local->loc1,
-					    &local->loc2);
-				if (!--callcnt)
-					break;
-			}
-		}
-	} else {
-		/* file doesn't seem to be present in storage nodes */
-		gf_log (this->name, GF_LOG_CRITICAL,
-			"CRITICAL: source file not in storage node, "
-			"rename successful on namespace :O");
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, -1, EIO, NULL,
-                              NULL, NULL, /* preoldparent, postoldparent */
-                              NULL, NULL); /* prenewparent, postnewparent */
-	}
-	return 0;
-}
-
-
-/**
- * unify_rename - One of the tricky function. The deadliest of all :O
- */
-int32_t
-unify_rename (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *oldloc,
-	      loc_t *newloc)
-{
-	unify_local_t *local = NULL;
-  	uint64_t tmp_list = 0;
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	loc_copy (&local->loc1, oldloc);
-	loc_copy (&local->loc2, newloc);
-
-	if ((local->loc1.path == NULL) || 
-	    (local->loc2.path == NULL)) {
-		gf_log (this->name, GF_LOG_CRITICAL, "Not enough memory :O");
-		STACK_UNWIND (frame, -1, ENOMEM, NULL,
-                              NULL, NULL, /* preoldparent, postoldparent */
-                              NULL, NULL); /* prenewparent, postnewparent */
-		return 0;
-	}
-  
-	inode_ctx_get (oldloc->inode, this, &tmp_list);
-	local->list = (int16_t *)(long)tmp_list;
-
-	STACK_WIND (frame,
-		    unify_ns_rename_cbk,
-		    NS(this),
-		    NS(this)->fops->rename,
-		    oldloc,
-		    newloc);
-	return 0;
-}
-
-/**
- * unify_link_cbk -
- */
-int32_t
-unify_link_cbk (call_frame_t *frame,
-		void *cookie,
-		xlator_t *this,
-		int32_t op_ret,
-		int32_t op_errno,
-		inode_t *inode,
-                struct iatt *buf,
-                struct iatt *preparent,
-                struct iatt *postparent)
-{
-	unify_local_t *local = frame->local;
-
-	if (op_ret >= 0) 
-		local->stbuf = *buf;
-	local->stbuf.ia_ino = local->ia_ino;
-
-	unify_local_wipe (local);
-	STACK_UNWIND (frame, op_ret, op_errno, inode, &local->stbuf,
-                      &local->oldpreparent, &local->oldpostparent);
-
-	return 0;
-}
-
-/**
- * unify_ns_link_cbk - 
- */
-int32_t
-unify_ns_link_cbk (call_frame_t *frame,
-		   void *cookie,
-		   xlator_t *this,
-		   int32_t op_ret,
-		   int32_t op_errno,
-		   inode_t *inode,
-                   struct iatt *buf,
-                   struct iatt *preparent,
-                   struct iatt *postparent)
-{
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-	int16_t *list = local->list;
-	int16_t index = 0;
-
-	if (op_ret == -1) {
-		/* No need to send link request to other servers, 
-		 * as namespace action failed 
-		 */
-		gf_log (this->name, GF_LOG_ERROR, 
-			"namespace: path(%s -> %s): %s", 
-			local->loc1.path, local->loc2.path, 
-			strerror (op_errno));
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, op_ret, op_errno, inode, buf,
-                              preparent, postparent);
-		return 0;
-	}
-
-	/* Update inode for this entry */
-	local->op_ret = 0;
-	local->ia_ino = buf->ia_ino;
-
-        local->oldpreparent = *preparent;
-        local->oldpostparent = *postparent;
-
-	/* Send link request to the node now */
-	for (index = 0; list[index] != -1; index++) {
-		char need_break = (list[index+1] == -1);
-		if (priv->xl_array[list[index]] != NS (this)) {
-			STACK_WIND (frame,
-				    unify_link_cbk,
-				    priv->xl_array[list[index]],
-				    priv->xl_array[list[index]]->fops->link,
-				    &local->loc1,
-				    &local->loc2);
-                        break;
-		}
-		if (need_break)
-			break;
-	}
-
-	return 0;
-}
-
-/**
- * unify_link - 
- */
-int32_t
-unify_link (call_frame_t *frame,
-	    xlator_t *this,
-	    loc_t *oldloc,
-	    loc_t *newloc)
-{
-	unify_local_t *local = NULL;
-  	uint64_t tmp_list = 0;
-
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (oldloc);
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (newloc);
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-
-	loc_copy (&local->loc1, oldloc);
-	loc_copy (&local->loc2, newloc);
-
-	inode_ctx_get (oldloc->inode, this, &tmp_list);
-	local->list = (int16_t *)(long)tmp_list;
-
-	STACK_WIND (frame,
-		    unify_ns_link_cbk,
-		    NS(this),
-		    NS(this)->fops->link,
-		    oldloc,
-		    newloc);
-
-	return 0;
-}
-
-
-/**
- * unify_checksum_cbk - 
- */
-int32_t
-unify_checksum_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    uint8_t *fchecksum,
-		    uint8_t *dchecksum)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, fchecksum, dchecksum);
-
-	return 0;
-}
-
-/**
- * unify_checksum - 
- */
-int32_t
-unify_checksum (call_frame_t *frame,
-		xlator_t *this,
-		loc_t *loc,
-		int32_t flag)
-{
-	STACK_WIND (frame,
-		    unify_checksum_cbk,
-		    NS(this),
-		    NS(this)->fops->checksum,
-		    loc,
-		    flag);
-
-	return 0;
-}
-
-
-/**
- * unify_finodelk_cbk - 
- */
-int
-unify_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		    int32_t op_ret, int32_t op_errno)
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-/**
- * unify_finodelk
- */
-int
-unify_finodelk (call_frame_t *frame, xlator_t *this,
-		const char *volume, fd_t *fd, int cmd, struct gf_flock *flock)
-{
-	UNIFY_CHECK_FD_CTX_AND_UNWIND_ON_ERR (fd);
-	xlator_t *child = NULL;
-	uint64_t tmp_child = 0;
-
-	fd_ctx_get (fd, this, &tmp_child);
-	child = (xlator_t *)(long)tmp_child;		     
-
-	STACK_WIND (frame, unify_finodelk_cbk,
-		    child, child->fops->finodelk,
-		    volume, fd, cmd, flock);
-
-	return 0;
-}
-
-
-
-/**
- * unify_fentrylk_cbk - 
- */
-int
-unify_fentrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		    int32_t op_ret, int32_t op_errno)
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-/**
- * unify_fentrylk
- */
-int
-unify_fentrylk (call_frame_t *frame, xlator_t *this,
-		const char *volume, fd_t *fd, const char *basename,
-		entrylk_cmd cmd, entrylk_type type)
-		
-{
-	UNIFY_CHECK_FD_CTX_AND_UNWIND_ON_ERR (fd);
-	xlator_t *child = NULL;
-	uint64_t tmp_child = 0;
-
-	fd_ctx_get (fd, this, &tmp_child);
-	child = (xlator_t *)(long)tmp_child;		     
-
-	STACK_WIND (frame, unify_fentrylk_cbk,
-		    child, child->fops->fentrylk,
-		    volume, fd, basename, cmd, type);
-
-	return 0;
-}
-
-
-
-/**
- * unify_fxattrop_cbk - 
- */
-int
-unify_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		    int32_t op_ret, int32_t op_errno, dict_t *xattr)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, xattr);
-	return 0;
-}
-
-/**
- * unify_fxattrop
- */
-int
-unify_fxattrop (call_frame_t *frame, xlator_t *this,
-		fd_t *fd, gf_xattrop_flags_t optype, dict_t *xattr)
-{
-	UNIFY_CHECK_FD_CTX_AND_UNWIND_ON_ERR (fd);
-	xlator_t *child = NULL;
-	uint64_t tmp_child = 0;
-
-	fd_ctx_get (fd, this, &tmp_child);
-	child = (xlator_t *)(long)tmp_child;		     
-
-	STACK_WIND (frame, unify_fxattrop_cbk,
-		    child, child->fops->fxattrop,
-		    fd, optype, xattr);
-
-	return 0;
-}
-
-
-/**
- * unify_inodelk_cbk - 
- */
-int
-unify_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		   int32_t op_ret, int32_t op_errno)
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-
-/**
- * unify_inodelk
- */
-int
-unify_inodelk (call_frame_t *frame, xlator_t *this,
-	       const char *volume, loc_t *loc, int cmd, struct gf_flock *flock)
-{
-	xlator_t *child = NULL;
-
-	child = unify_loc_subvol (loc, this);
-
-	STACK_WIND (frame, unify_inodelk_cbk,
-		    child, child->fops->inodelk,
-		    volume, loc, cmd, flock);
-
-	return 0;
-}
-
-
-
-/**
- * unify_entrylk_cbk - 
- */
-int
-unify_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		   int32_t op_ret, int32_t op_errno)
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-/**
- * unify_entrylk
- */
-int
-unify_entrylk (call_frame_t *frame, xlator_t *this,
-	       const char *volume, loc_t *loc, const char *basename,
-	       entrylk_cmd cmd, entrylk_type type)
-		
-{
-	xlator_t *child = NULL;
-
-	child = unify_loc_subvol (loc, this);
-
-	STACK_WIND (frame, unify_entrylk_cbk,
-		    child, child->fops->entrylk,
-		    volume, loc, basename, cmd, type);
-
-	return 0;
-}
-
-
-
-/**
- * unify_xattrop_cbk - 
- */
-int
-unify_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		   int32_t op_ret, int32_t op_errno, dict_t *xattr)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, xattr);
-	return 0;
-}
-
-/**
- * unify_xattrop
- */
-int
-unify_xattrop (call_frame_t *frame, xlator_t *this,
-		loc_t *loc, gf_xattrop_flags_t optype, dict_t *xattr)
-{
-	xlator_t *child = NULL;
-
-	child = unify_loc_subvol (loc, this);
-
-	STACK_WIND (frame, unify_xattrop_cbk,
-		    child, child->fops->xattrop,
-		    loc, optype, xattr);
-
-	return 0;
-}
-
-int
-unify_forget (xlator_t *this,
-              inode_t *inode)
-{
-        int16_t *list = NULL;
-        uint64_t tmp_list = 0;
-
-        if (inode->ia_type && (!IA_ISDIR(inode->ia_type))) {
-                inode_ctx_get (inode, this, &tmp_list);
-                if (tmp_list) {
-                        list = (int16_t *)(long)tmp_list;
-                        GF_FREE (list);
-                }
-        }
-
-        return 0;
-}
-
-/**
- * notify
- */
-int32_t
-notify (xlator_t *this,
-        int32_t event,
-        void *data,
-        ...)
-{
-	unify_private_t *priv = this->private;
-	struct sched_ops *sched = NULL;
-
-	if (!priv) {
-		return 0;
-	}
-
-	sched = priv->sched_ops;    
-	if (!sched) {
-		gf_log (this->name, GF_LOG_CRITICAL, "No scheduler :O");
-		raise (SIGTERM);
-		return 0;
-	}
-	if (priv->namespace == data) {
-		if (event == GF_EVENT_CHILD_UP) {
-			sched->notify (this, event, data);
-		}
-		return 0;
-	}
-
-	switch (event)
-	{
-	case GF_EVENT_CHILD_UP:
-	{
-		/* Call scheduler's update () to enable it for scheduling */
-		sched->notify (this, event, data);
-	
-		LOCK (&priv->lock);
-		{
-			/* Increment the inode's generation, which is 
-			   used for self_heal */
-			++priv->inode_generation;
-			++priv->num_child_up;
-		}
-		UNLOCK (&priv->lock);
-
-		if (!priv->is_up) {
-			default_notify (this, event, data);
-			priv->is_up = 1;
-		}
-	}
-	break;
-	case GF_EVENT_CHILD_DOWN:
-	{
-		/* Call scheduler's update () to disable the child node 
-		 * for scheduling
-		 */
-		sched->notify (this, event, data);
-		LOCK (&priv->lock);
-		{
-			--priv->num_child_up;
-		}
-		UNLOCK (&priv->lock);
-
-		if (priv->num_child_up == 0) {
-			/* Send CHILD_DOWN to upper layer */
-			default_notify (this, event, data);
-			priv->is_up = 0;
-		}
-	}
-	break;
-
-	default:
-	{
-		default_notify (this, event, data);
-	}
-	break;
-	}
-
-	return 0;
-}
-
-int32_t
-mem_acct_init (xlator_t *this)
-{
-        int     ret = -1;
-
-        if (!this)
-                return ret;
-
-        ret = xlator_mem_acct_init (this, gf_unify_mt_end + 1);
-        
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
-                                "failed");
-                return ret;
-        }
-
-        return ret;
-}
-
-/** 
- * init - This function is called first in the xlator, while initializing.
- *   All the config file options are checked and appropriate flags are set.
- *
- * @this - 
- */
-int32_t 
-init (xlator_t *this)
-{
-	int32_t          ret       = 0;
-	int32_t          count     = 0;
-	data_t          *scheduler = NULL;
-	data_t          *data      = NULL;
-	xlator_t        *ns_xl     = NULL;
-	xlator_list_t   *trav      = NULL;
-	xlator_list_t   *xlparent  = NULL;
-	xlator_list_t   *parent    = NULL;
-	unify_private_t *_private  = NULL; 
-
-
-	/* Check for number of child nodes, if there is no child nodes, exit */
-	if (!this->children) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"No child nodes specified. check \"subvolumes \" "
-			"option in volfile");
-		return -1;
-	}
-
-  	if (!this->parents) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"dangling volume. check volfile ");
-	}
-  
-	/* Check for 'scheduler' in volume */
-	scheduler = dict_get (this->options, "scheduler");
-	if (!scheduler) {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"\"option scheduler <x>\" is missing in volfile");
-		return -1;
-	}
-
-	/* Setting "option namespace <node>" */
-	data = dict_get (this->options, "namespace");
-	if(!data) {
-		gf_log (this->name, GF_LOG_CRITICAL, 
-			"namespace option not specified, Exiting");
-		return -1;
-	}
-	/* Search namespace in the child node, if found, exit */
-	trav = this->children;
-	while (trav) {
-		if (strcmp (trav->xlator->name, data->data) == 0)
-			break;
-		trav = trav->next;
-	}
-	if (trav) {
-		gf_log (this->name, GF_LOG_CRITICAL, 
-			"namespace node used as a subvolume, Exiting");
-		return -1;
-	}
-	
-	/* Search for the namespace node, if found, continue */
-	ns_xl = this->next;
-	while (ns_xl) {
-		if (strcmp (ns_xl->name, data->data) == 0)
-			break;
-		ns_xl = ns_xl->next;
-	}
-	if (!ns_xl) {
-		gf_log (this->name, GF_LOG_CRITICAL, 
-			"namespace node not found in volfile, Exiting");
-		return -1;
-	}
-	
-	gf_log (this->name, GF_LOG_DEBUG, 
-		"namespace node specified as %s", data->data);
-	
-	_private = GF_CALLOC (1, sizeof (*_private), 
-                              gf_unify_mt_unify_private_t);
-	ERR_ABORT (_private);
-	_private->sched_ops = get_scheduler (this, scheduler->data);
-	if (!_private->sched_ops) {
-		gf_log (this->name, GF_LOG_CRITICAL, 
-			"Error while loading scheduler. Exiting");
-		GF_FREE (_private);
-		return -1;
-	}
-	
-	if (ns_xl->parents) {
-		gf_log (this->name, GF_LOG_CRITICAL,
-			"Namespace node should not be a child of any other node. Exiting");
-		GF_FREE (_private);
-		return -1;
-	}
-
-	_private->namespace = ns_xl;
-	
-	/* update _private structure */
-	{
-		count = 0;
-		trav = this->children;
-		/* Get the number of child count */
-		while (trav) {
-			count++;
-			trav = trav->next;
-		}
-		
-		gf_log (this->name, GF_LOG_DEBUG, 
-			"Child node count is %d", count);    
-
-		_private->child_count = count;
-		if (count == 1) {
-			/* TODO: Should I error out here? */
-			gf_log (this->name, GF_LOG_CRITICAL, 
-				"WARNING: You have defined only one "
-				"\"subvolumes\" for unify volume. It may not "
-				"be the desired config, review your volume "
-				"volfile. If this is how you are testing it,"
-				" you may hit some performance penalty");
-		}
-		
-		_private->xl_array = GF_CALLOC (1, 
-					        sizeof (xlator_t) * (count + 1),
-                                                gf_unify_mt_xlator_t);
-		ERR_ABORT (_private->xl_array);
-		
-		count = 0;
-		trav = this->children;
-		while (trav) {
-			_private->xl_array[count++] = trav->xlator;
-			trav = trav->next;
-		}
-		_private->xl_array[count] = _private->namespace;
-		
-		/* self-heal part, start with generation '1' */
-		_private->inode_generation = 1; 
-                /* Because, Foreground part is tested well */
-		_private->self_heal = ZR_UNIFY_FG_SELF_HEAL; 
-		data = dict_get (this->options, "self-heal");
-		if (data) {
-			if (strcasecmp (data->data, "off") == 0) 
-				_private->self_heal = ZR_UNIFY_SELF_HEAL_OFF;
-
-			if (strcasecmp (data->data, "foreground") == 0)
-				_private->self_heal = ZR_UNIFY_FG_SELF_HEAL;
-
-			if (strcasecmp (data->data, "background") == 0)
-				_private->self_heal = ZR_UNIFY_BG_SELF_HEAL;
-		}
-    
-		/* optimist - ask bulde for more about it */
-		data = dict_get (this->options, "optimist");
-		if (data) {
-			if (gf_string2boolean (data->data, 
-					       &_private->optimist) == -1) {
-				gf_log (this->name, GF_LOG_ERROR, 
-					"optimist excepts only boolean "
-					"options");
-			}
-		}
-
-		LOCK_INIT (&_private->lock);
-	}
-
-	/* Now that everything is fine. */
-	this->private = (void *)_private;
-	{
-                ret = _private->sched_ops->mem_acct_init (this);
-
-                if (ret == -1) {
-                        return -1;
-                }
-
-		/* Initialize scheduler, if everything else is successful */
-		ret = _private->sched_ops->init (this); 
-		if (ret == -1) {
-			gf_log (this->name, GF_LOG_CRITICAL,
-				"Initializing scheduler failed, Exiting");
-			GF_FREE (_private);
-			return -1;
-		}
-
-
-		ret = 0;
-
-		/* This section is required because some fops may look 
-		 * for 'xl->parent' variable 
-		 */
-		xlparent = GF_CALLOC (1, sizeof (*xlparent),
-                                      gf_unify_mt_xlator_list_t);
-		xlparent->xlator = this;
-		if (!ns_xl->parents) {
-			ns_xl->parents = xlparent;
-		} else {
-			parent = ns_xl->parents;
-			while (parent->next)
-				parent = parent->next;
-			parent->next = xlparent;
-		}
-	}
-
-	/* Tell namespace node that init is done */
-	xlator_notify (ns_xl, GF_EVENT_PARENT_UP, this);
-
-	return 0;
-}
-
-/** 
- * fini  - Free all the allocated memory 
- */
-void
-fini (xlator_t *this)
-{
-	unify_private_t *priv = this->private;
-	priv->sched_ops->fini (this);
-	this->private = NULL;
-	LOCK_DESTROY (&priv->lock);
-	GF_FREE (priv->xl_array);
-	GF_FREE (priv);
-	return;
-}
-
-
-struct xlator_fops fops = {
-	.stat        = unify_stat,
-	.readlink    = unify_readlink,
-	.mknod       = unify_mknod,
-	.mkdir       = unify_mkdir,
-	.unlink      = unify_unlink,
-	.rmdir       = unify_rmdir,
-	.symlink     = unify_symlink,
-	.rename      = unify_rename,
-	.link        = unify_link,
-	.truncate    = unify_truncate,
-	.create      = unify_create,
-	.open        = unify_open,
-	.readv       = unify_readv,
-	.writev      = unify_writev,
-	.statfs      = unify_statfs,
-	.flush       = unify_flush,
-	.fsync       = unify_fsync,
-	.setxattr    = unify_setxattr,
-	.getxattr    = unify_getxattr,
-	.removexattr = unify_removexattr,
-	.opendir     = unify_opendir,
-	.readdir     = unify_readdir,
-	.readdirp    = unify_readdirp,
-	.fsyncdir    = unify_fsyncdir,
-	.access      = unify_access,
-	.ftruncate   = unify_ftruncate,
-	.fstat       = unify_fstat,
-	.lk          = unify_lk,
-	.lookup      = unify_lookup,
-	.getdents    = unify_getdents,
-	.checksum    = unify_checksum,
-	.inodelk     = unify_inodelk,
-	.finodelk    = unify_finodelk,
-	.entrylk     = unify_entrylk,
-	.fentrylk    = unify_fentrylk,
-	.xattrop     = unify_xattrop,
-	.fxattrop    = unify_fxattrop,
-        .setattr     = unify_setattr, 
-        .fsetattr    = unify_fsetattr,
-};
-
-
-struct xlator_cbks cbks = {
-        .forget  = unify_forget,
-};
-
-struct volume_options options[] = {
-	{ .key   = { "namespace" },  
-	  .type  = GF_OPTION_TYPE_XLATOR 
-	},
-	{ .key   = { "scheduler" },  
-	  .value = { "alu", "rr", "random", "nufa", "switch" },
-	  .type  = GF_OPTION_TYPE_STR
-	},
-	{ .key   = {"self-heal"},  
-	  .value = { "foreground", "background", "off" },
-	  .type  = GF_OPTION_TYPE_STR
-	},
-	/* TODO: remove it some time later */
-	{ .key   = {"optimist"},  
-	  .type  = GF_OPTION_TYPE_BOOL 
-	},
-
-	{ .key   = {NULL} },
-};
diff --git a/xlators/cluster/unify/src/unify.h b/xlators/cluster/unify/src/unify.h
deleted file mode 100644
index dbd5e44a2..000000000
--- a/xlators/cluster/unify/src/unify.h
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#ifndef _UNIFY_H
-#define _UNIFY_H
-
-#include "scheduler.h"
-#include "list.h"
-#include "unify-mem-types.h"
-
-#define MAX_DIR_ENTRY_STRING     (32 * 1024)
-
-#define ZR_UNIFY_SELF_HEAL_OFF 0
-#define ZR_UNIFY_FG_SELF_HEAL  1
-#define ZR_UNIFY_BG_SELF_HEAL  2
-
-/* Sometimes one should use completely random numbers.. its good :p */
-#define UNIFY_SELF_HEAL_GETDENTS_COUNT 512
-
-#define NS(xl)          (((unify_private_t *)xl->private)->namespace)
-
-/* This is used to allocate memory for local structure */
-#define INIT_LOCAL(fr, loc)                   \
-do {                                          \
-  loc = GF_CALLOC (1, sizeof (unify_local_t), gf_unify_mt_unify_local_t);   \
-  ERR_ABORT (loc);			      \
-  if (!loc) {                                 \
-    STACK_UNWIND (fr, -1, ENOMEM);            \
-    return 0;                                 \
-  }                                           \
-  fr->local = loc;                            \
-  loc->op_ret = -1;                           \
-  loc->op_errno = ENOENT;                     \
-} while (0)
-
-
-
-struct unify_private {
-	/* Update this structure depending on requirement */
-	void *scheduler;               /* THIS SHOULD BE THE FIRST VARIABLE, 
-					  if xlator is using scheduler */
-	struct sched_ops *sched_ops;   /* Scheduler options  */
-	xlator_t *namespace;           /* ptr to namespace xlator */
-	xlator_t **xl_array;
-	gf_boolean_t optimist;
-	int16_t child_count;
-	int16_t num_child_up;
-	uint8_t self_heal;
-	uint8_t is_up;
-	uint64_t inode_generation;
-	gf_lock_t lock;
-};
-typedef struct unify_private unify_private_t;
-
-struct unify_self_heal_struct {
-	uint8_t dir_checksum[NAME_MAX];
-	uint8_t ns_dir_checksum[NAME_MAX];
-	uint8_t file_checksum[NAME_MAX];
-	uint8_t ns_file_checksum[NAME_MAX];
-	off_t *offset_list;
-	int   *count_list;
-	dir_entry_t **entry_list;
-};
-
-
-struct _unify_local_t {
-	int32_t call_count;
-	int32_t op_ret;
-	int32_t op_errno;
-	mode_t mode;
-	off_t offset;
-	dev_t dev;
-	uid_t uid;
-	gid_t gid;
-	int32_t flags;
-	int32_t entry_count;
-	int32_t count;    // dir_entry_t count;
-	fd_t *fd;
-	struct iatt stbuf;
-        struct iatt stpre;
-        struct iatt stpost;
-	struct statvfs statvfs_buf;
-	struct timespec tv[2];
-	char *name;
-	int32_t revalidate;
-
-	ino_t ia_ino;
-	nlink_t ia_nlink;
-  
-	dict_t *dict;
-
-	int16_t *list;
-	int16_t *new_list; /* Used only in case of rename */
-	int16_t index;
-
-	int32_t failed;
-	int32_t return_eio;  /* Used in case of different st-mode 
-				present for a given path */
-
-	uint64_t inode_generation; /* used to store the per directory 
-				    * inode_generation. Got from inode's ctx 
-				    * of directory inodes
-				    */
-
-	struct unify_self_heal_struct *sh_struct;
-	loc_t loc1, loc2;
-
-        struct iatt poststbuf;
-        /* When not used for rename, old*
-         * are used as the attrs for the current
-         * parent directory.
-         */
-        struct iatt oldpreparent;
-        struct iatt oldpostparent;
-        struct iatt newpreparent;
-        struct iatt newpostparent;
-        int32_t wbflags;
-};
-typedef struct _unify_local_t unify_local_t;
-
-int32_t zr_unify_self_heal (call_frame_t *frame,
-			    xlator_t *this,
-			    unify_local_t *local);
-
-#endif /* _UNIFY_H */
diff --git a/xlators/debug/error-gen/src/Makefile.am b/xlators/debug/error-gen/src/Makefile.am
index df9080358..5075c59a8 100644
--- a/xlators/debug/error-gen/src/Makefile.am
+++ b/xlators/debug/error-gen/src/Makefile.am
@@ -2,15 +2,16 @@
 xlator_LTLIBRARIES = error-gen.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/debug
 
-error_gen_la_LDFLAGS = -module -avoidversion
+error_gen_la_LDFLAGS = -module -avoid-version
 
 error_gen_la_SOURCES = error-gen.c
 error_gen_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = error-gen.h error-gen-mem-types.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)\
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES = 
 
diff --git a/xlators/debug/error-gen/src/error-gen-mem-types.h b/xlators/debug/error-gen/src/error-gen-mem-types.h
index b643dc5f7..f02280535 100644
--- a/xlators/debug/error-gen/src/error-gen-mem-types.h
+++ b/xlators/debug/error-gen/src/error-gen-mem-types.h
@@ -1,23 +1,13 @@
 /*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef __ERROR_GEN_MEM_TYPES_H__
 #define __ERROR_GEN_MEM_TYPES_H__
 
diff --git a/xlators/debug/error-gen/src/error-gen.c b/xlators/debug/error-gen/src/error-gen.c
index 6ecca2ee5..ec0874b35 100644
--- a/xlators/debug/error-gen/src/error-gen.c
+++ b/xlators/debug/error-gen/src/error-gen.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -24,6 +14,7 @@
 
 #include "xlator.h"
 #include "error-gen.h"
+#include "statedump.h"
 
 sys_error_t error_no_list[] = {
         [GF_FOP_LOOKUP]            = { .error_no_count = 4,
@@ -91,9 +82,10 @@ sys_error_t error_no_list[] = {
         [GF_FOP_READ]              = { .error_no_count = 5,
                                     .error_no = {EINVAL,EBADF,EFAULT,EISDIR,
                                                  ENAMETOOLONG}},
-        [GF_FOP_WRITE]             = { .error_no_count = 5,
+        [GF_FOP_WRITE]             = { .error_no_count = 7,
                                     .error_no = {EINVAL,EBADF,EFAULT,EISDIR,
-                                                 ENAMETOOLONG}},
+                                                 ENAMETOOLONG,ENOSPC,
+						 GF_ERROR_SHORT_WRITE}},
         [GF_FOP_STATFS]            = {.error_no_count = 10,
                                    .error_no = {EACCES,EBADF,EFAULT,EINTR,
                                                 EIO,ENAMETOOLONG,ENOENT,
@@ -246,6 +238,8 @@ conv_errno_to_int (char **error_no)
                 return EINTR;
         else if (!strcmp ((*error_no), "EFBIG"))
                 return EFBIG;
+	else if (!strcmp((*error_no), "GF_ERROR_SHORT_WRITE"))
+		return GF_ERROR_SHORT_WRITE;
         else
                 return EAGAIN;
 }
@@ -1114,10 +1108,26 @@ error_gen_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
         if (enable)
                 op_errno = error_gen (this, GF_FOP_WRITE);
 
-	if (op_errno) {
+	if (op_errno == GF_ERROR_SHORT_WRITE) {
+		struct iovec *shortvec;
+
+		/*
+		 * A short write error returns some value less than what was
+		 * requested from a write. To simulate this, replace the vector
+		 * with one half the size;
+		 */
+		shortvec = iov_dup(vector, 1);
+		shortvec->iov_len /= 2;
+
+		STACK_WIND(frame, error_gen_writev_cbk, FIRST_CHILD(this),
+			   FIRST_CHILD(this)->fops->writev, fd, shortvec, count,
+			   off, flags, iobref, xdata);
+		GF_FREE(shortvec);
+		return 0;
+	} else if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
 		STACK_UNWIND_STRICT (writev, frame, -1, op_errno, NULL, NULL, xdata);
-        return 0;
+        	return 0;
 	}
 
 	STACK_WIND (frame, error_gen_writev_cbk,
@@ -1944,6 +1954,85 @@ error_gen_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
 	return 0;
 }
 
+static void
+error_gen_set_failure (eg_t *pvt, int percent)
+{
+        GF_ASSERT (pvt);
+
+        if (percent)
+                pvt->failure_iter_no = 100/percent;
+        else
+                pvt->failure_iter_no = 100/GF_FAILURE_DEFAULT;
+}
+
+static void
+error_gen_parse_fill_fops (eg_t *pvt, char *enable_fops)
+{
+        char            *op_no_str = NULL;
+        int              op_no = -1;
+        int              i = 0;
+        xlator_t        *this = THIS;
+        char            *saveptr = NULL;
+
+        GF_ASSERT (pvt);
+        GF_ASSERT (this);
+
+        for (i = 0; i < GF_FOP_MAXVALUE; i++)
+                pvt->enable[i] = 0;
+
+        if (!enable_fops) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "All fops are enabled.");
+                for (i = 0; i < GF_FOP_MAXVALUE; i++)
+                        pvt->enable[i] = 1;
+        } else {
+                op_no_str = strtok_r (enable_fops, ",", &saveptr);
+                while (op_no_str) {
+                        op_no = get_fop_int (&op_no_str);
+                        if (op_no == -1) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "Wrong option value %s", op_no_str);
+                        } else
+                                pvt->enable[op_no] = 1;
+
+                        op_no_str = strtok_r (NULL, ",", &saveptr);
+                }
+        }
+}
+
+int32_t
+error_gen_priv_dump (xlator_t *this)
+{
+        char            key_prefix[GF_DUMP_MAX_BUF_LEN];
+        int             ret = -1;
+        eg_t            *conf = NULL;
+
+        if (!this)
+            goto out;
+
+        conf = this->private;
+        if (!conf)
+            goto out;
+
+        ret = TRY_LOCK(&conf->lock);
+        if (ret != 0) {
+                return ret;
+        }
+
+        gf_proc_dump_add_section("xlator.debug.error-gen.%s.priv", this->name);
+        gf_proc_dump_build_key(key_prefix,"xlator.debug.error-gen","%s.priv",
+                               this->name);
+
+        gf_proc_dump_write("op_count", "%d", conf->op_count);
+        gf_proc_dump_write("failure_iter_no", "%d", conf->failure_iter_no);
+        gf_proc_dump_write("error_no", "%s", conf->error_no);
+        gf_proc_dump_write("random_failure", "%d", conf->random_failure);
+
+        UNLOCK(&conf->lock);
+out:
+        return ret;
+}
+
 int32_t
 mem_acct_init (xlator_t *this)
 {
@@ -1964,18 +2053,43 @@ mem_acct_init (xlator_t *this)
 }
 
 int
+reconfigure (xlator_t *this, dict_t *options)
+{
+        eg_t            *pvt = NULL;
+        int32_t          ret = 0;
+        char            *error_enable_fops = NULL;
+        int32_t          failure_percent_int = 0;
+
+        if (!this || !this->private)
+                goto out;
+
+        pvt = this->private;
+
+        GF_OPTION_RECONF ("error-no", pvt->error_no, options, str, out);
+
+        GF_OPTION_RECONF ("failure", failure_percent_int, options, int32,
+                          out);
+
+        GF_OPTION_RECONF ("enable", error_enable_fops, options, str, out);
+
+        GF_OPTION_RECONF ("random-failure", pvt->random_failure, options,
+                          bool, out);
+
+        error_gen_parse_fill_fops (pvt, error_enable_fops);
+        error_gen_set_failure (pvt, failure_percent_int);
+
+        ret = 0;
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "reconfigure returning %d", ret);
+        return ret;
+}
+
+int
 init (xlator_t *this)
 {
         eg_t            *pvt = NULL;
-        data_t          *error_no = NULL;
-        data_t          *failure_percent = NULL;
-        data_t          *enable = NULL;
-        gf_boolean_t    random_failure = _gf_false;
         int32_t          ret = 0;
         char            *error_enable_fops = NULL;
-        char            *op_no_str = NULL;
-        int              op_no = -1;
-        int              i = 0;
         int32_t          failure_percent_int = 0;
 
         if (!this->children || this->children->next) {
@@ -1990,79 +2104,34 @@ init (xlator_t *this)
                         "dangling volume. check volfile ");
         }
 
-        error_no = dict_get (this->options, "error-no");
-        failure_percent = dict_get (this->options, "failure");
-        enable = dict_get (this->options, "enable");
-
         pvt = GF_CALLOC (1, sizeof (eg_t), gf_error_gen_mt_eg_t);
 
         if (!pvt) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "out of memory.");
                 ret = -1;
                 goto out;
         }
 
         LOCK_INIT (&pvt->lock);
 
-        for (i = 0; i < GF_FOP_MAXVALUE; i++)
-                pvt->enable[i] = 0;
-        if (!error_no) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "error-no not specified.");
-        } else {
-                pvt->error_no = data_to_str (error_no);
-        }
+        GF_OPTION_INIT ("error-no", pvt->error_no, str, out);
 
-        if (!failure_percent) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "failure percent not specified.");
-                pvt->failure_iter_no = 100/GF_FAILURE_DEFAULT;
-        } else {
-                failure_percent_int = data_to_int32 (failure_percent);
-                if (failure_percent_int)
-                        pvt->failure_iter_no = 100/failure_percent_int;
-                else
-                        pvt->failure_iter_no = 100/GF_FAILURE_DEFAULT;
-        }
+        GF_OPTION_INIT ("failure", failure_percent_int, int32, out);
 
-        if (!enable) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "All fops are enabled.");
-                for (i = 0; i < GF_FOP_MAXVALUE; i++)
-                        pvt->enable[i] = 1;
-        } else {
-                error_enable_fops = data_to_str (enable);
-                op_no_str = error_enable_fops;
-                while ((*error_enable_fops) != '\0') {
-                        error_enable_fops++;
-                        if (((*error_enable_fops) == ',') ||
-                            ((*error_enable_fops) == '\0')) {
-                                if ((*error_enable_fops) != '\0') {
-                                        (*error_enable_fops) = '\0';
-                                        error_enable_fops++;
-                                }
-                                op_no = get_fop_int (&op_no_str);
-                                if (op_no == -1) {
-                                        gf_log (this->name, GF_LOG_WARNING,
-                                                "Wrong option value %s",
-                                                op_no_str);
-                                } else
-                                        pvt->enable[op_no] = 1;
-                                op_no_str = error_enable_fops;
-                        }
-                }
-        }
+        GF_OPTION_INIT ("enable", error_enable_fops, str, out);
 
-        random_failure = dict_get_str_boolean (this->options, "random-failure",
-                                               _gf_false);
-        pvt->random_failure = random_failure;
+        GF_OPTION_INIT ("random-failure", pvt->random_failure, bool, out);
+
+
+        error_gen_parse_fill_fops (pvt, error_enable_fops);
+        error_gen_set_failure (pvt, failure_percent_int);
 
         this->private = pvt;
 
         /* Give some seed value here */
         srand (time(NULL));
 out:
+        if (ret)
+                GF_FREE (pvt);
         return ret;
 }
 
@@ -2084,9 +2153,12 @@ fini (xlator_t *this)
         return;
 }
 
-struct xlator_fops cbks = {
+struct xlator_dumpops dumpops = {
+        .priv = error_gen_priv_dump,
 };
 
+struct xlator_fops cbks;
+
 struct xlator_fops fops = {
 	.lookup      = error_gen_lookup,
 	.stat        = error_gen_stat,
@@ -2134,17 +2206,27 @@ struct xlator_fops fops = {
 
 struct volume_options options[] = {
         { .key  = {"failure"},
-          .type = GF_OPTION_TYPE_INT },
+          .type = GF_OPTION_TYPE_INT,
+          .description = "Percentage failure of operations when enabled.",
+        },
+
         { .key  = {"error-no"},
           .value = {"ENOENT","ENOTDIR","ENAMETOOLONG","EACCES","EBADF",
                     "EFAULT","ENOMEM","EINVAL","EIO","EEXIST","ENOSPC",
                     "EPERM","EROFS","EBUSY","EISDIR","ENOTEMPTY","EMLINK"
                     "ENODEV","EXDEV","EMFILE","ENFILE","ENOSYS","EINTR",
-                    "EFBIG","EAGAIN"},
-          .type = GF_OPTION_TYPE_STR },
+                    "EFBIG","EAGAIN","GF_ERROR_SHORT_WRITE"},
+          .type = GF_OPTION_TYPE_STR,
+        },
+
         { .key = {"random-failure"},
-          .type = GF_OPTION_TYPE_BOOL},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
+        },
+
         { .key  = {"enable"},
-          .type = GF_OPTION_TYPE_STR },
+          .type = GF_OPTION_TYPE_STR,
+        },
+
         { .key  = {NULL} }
 };
diff --git a/xlators/debug/error-gen/src/error-gen.h b/xlators/debug/error-gen/src/error-gen.h
index bb3adb2ab..d92c23062 100644
--- a/xlators/debug/error-gen/src/error-gen.h
+++ b/xlators/debug/error-gen/src/error-gen.h
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _ERROR_GEN_H
 #define _ERROR_GEN_H
 
@@ -29,6 +19,18 @@
 
 #define GF_FAILURE_DEFAULT 10
 
+/*
+ * Pseudo-errors refer to errors beyond the scope of traditional <-1, op_errno>
+ * returns. This facilitates the ability to return unexpected, but not -1 values
+ * and/or to inject operations that lead to implicit error conditions. The range
+ * for pseudo errors resides at a high value to avoid conflicts with the errno
+ * range.
+ */
+enum GF_PSEUDO_ERRORS {
+	GF_ERROR_SHORT_WRITE = 1000,	/* short writev return value */
+	GF_ERROR_MAX
+};
+
 typedef struct {
         int enable[GF_FOP_MAXVALUE];
         int op_count;
diff --git a/xlators/debug/io-stats/src/Makefile.am b/xlators/debug/io-stats/src/Makefile.am
index b894e79c3..dff294cd8 100644
--- a/xlators/debug/io-stats/src/Makefile.am
+++ b/xlators/debug/io-stats/src/Makefile.am
@@ -2,14 +2,17 @@
 xlator_LTLIBRARIES = io-stats.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/debug
 
-io_stats_la_LDFLAGS = -module -avoidversion
+io_stats_la_LDFLAGS = -module -avoid-version
 
 io_stats_la_SOURCES = io-stats.c
 io_stats_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = io-stats-mem-types.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)\
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src \
+	-I$(top_srcdir)/rpc/rpc-lib/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES =
diff --git a/xlators/debug/io-stats/src/io-stats-mem-types.h b/xlators/debug/io-stats/src/io-stats-mem-types.h
index 2063f6d6a..c30dfb17e 100644
--- a/xlators/debug/io-stats/src/io-stats-mem-types.h
+++ b/xlators/debug/io-stats/src/io-stats-mem-types.h
@@ -1,24 +1,13 @@
-
 /*
-   Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef __IO_STATS_MEM_TYPES_H__
 #define __IO_STATS_MEM_TYPES_H__
 
diff --git a/xlators/debug/io-stats/src/io-stats.c b/xlators/debug/io-stats/src/io-stats.c
index c59370c7a..9e48a7c6e 100644
--- a/xlators/debug/io-stats/src/io-stats.c
+++ b/xlators/debug/io-stats/src/io-stats.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -46,6 +36,7 @@
 #include <stdarg.h>
 #include "defaults.h"
 #include "logging.h"
+#include "cli1-xdr.h"
 
 #define MAX_LIST_MEMBERS 100
 
@@ -58,7 +49,7 @@ typedef enum {
         IOS_STATS_TYPE_READDIRP,
         IOS_STATS_TYPE_READ_THROUGHPUT,
         IOS_STATS_TYPE_WRITE_THROUGHPUT,
-        IOS_STATS_TYPE_MAX,
+        IOS_STATS_TYPE_MAX
 }ios_stats_type_t;
 
 typedef enum {
@@ -293,43 +284,43 @@ is_fop_latency_started (call_frame_t *frame)
                 }                                                               \
                 UNLOCK (&iosstat->lock);                                        \
                 ios_stat_add_to_list (&conf->list[type],                        \
-				     value, iosstat);                           \
+                                     value, iosstat);                           \
                                                                                 \
         } while (0)
 
 
 #define BUMP_THROUGHPUT(iosstat, type)						\
         do {									\
-	        struct ios_conf         *conf = NULL;				\
-		double                   elapsed;				\
-		struct timeval          *begin, *end;				\
-		double                   throughput;				\
+                struct ios_conf         *conf = NULL;				\
+                double                   elapsed;				\
+                struct timeval          *begin, *end;				\
+                double                   throughput;				\
                int                      flag = 0;                              \
-										\
-		begin = &frame->begin;						\
-		end   = &frame->end;						\
-										\
-		elapsed = (end->tv_sec - begin->tv_sec) * 1e6			\
-			+ (end->tv_usec - begin->tv_usec);			\
-		throughput = op_ret / elapsed;					\
-										\
-		conf = this->private;						\
-		LOCK(&iosstat->lock);						\
-		{								\
-			if (iosstat->thru_counters[type].throughput             \
+                                                                                \
+                begin = &frame->begin;						\
+                end   = &frame->end;						\
+                                                                                \
+                elapsed = (end->tv_sec - begin->tv_sec) * 1e6			\
+                        + (end->tv_usec - begin->tv_usec);			\
+                throughput = op_ret / elapsed;					\
+                                                                                \
+                conf = this->private;						\
+                LOCK(&iosstat->lock);						\
+                {								\
+                        if (iosstat->thru_counters[type].throughput             \
                                 <= throughput) {                                \
-				iosstat->thru_counters[type].throughput =       \
+                                iosstat->thru_counters[type].throughput =       \
                                                                 throughput;     \
-				gettimeofday (&iosstat->                        \
+                                gettimeofday (&iosstat->                        \
                                              thru_counters[type].time, NULL);   \
                                flag = 1;                                       \
                         }							\
                 }								\
-        	UNLOCK (&iosstat->lock);					\
+                UNLOCK (&iosstat->lock);					\
                if (flag)                                                       \
                        ios_stat_add_to_list (&conf->thru_list[type],           \
                                                throughput, iosstat);           \
-	} while (0)
+        } while (0)
 
 int
 ios_fd_ctx_get (fd_t *fd, xlator_t *this, struct ios_fd **iosfd)
@@ -484,12 +475,12 @@ ios_stat_add_to_list (struct ios_stat_head *list_head, uint64_t value,
                         new = GF_CALLOC (1, sizeof (*new),
                                         gf_io_stats_mt_ios_stat_list);
                         new->iosstat = iosstat;
-                        new->value = value; 
+                        new->value = value;
                         ios_stat_ref (iosstat);
-                        list_add_tail (&new->list, &tmp->list);  
+                        list_add_tail (&new->list, &tmp->list);
                         stat = last->iosstat;
                         last->iosstat = NULL;
-                        ios_stat_unref (stat); 
+                        ios_stat_unref (stat);
                         list_del (&last->list);
                         GF_FREE (last);
                         if (reposition == MAX_LIST_MEMBERS)
@@ -511,7 +502,7 @@ ios_stat_add_to_list (struct ios_stat_head *list_head, uint64_t value,
                         list_head->members++;
                         if (list_head->min_cnt > value)
                                 list_head->min_cnt = value;
-                } 
+                }
         }
 out:
         UNLOCK (&list_head->lock);
@@ -732,13 +723,13 @@ io_stats_dump_global_to_logfp (xlator_t *this, struct ios_global_stats *stats,
                 ios_log (this, logfp, "\nTIMESTAMP \t\t\t THROUGHPUT(KBPS)"
                          "\tFILE NAME");
                 list_head = &conf->thru_list[IOS_STATS_THRU_READ];
-		ios_dump_throughput_stats(list_head, this, logfp, IOS_STATS_THRU_READ);
+                ios_dump_throughput_stats(list_head, this, logfp, IOS_STATS_THRU_READ);
 
                 ios_log (this, logfp, "\n======Write Throughput File Stats======");
                 ios_log (this, logfp, "\nTIMESTAMP \t\t\t THROUGHPUT(KBPS)"
                          "\tFILE NAME");
                 list_head = &conf->thru_list[IOS_STATS_THRU_WRITE];
-		ios_dump_throughput_stats (list_head, this, logfp, IOS_STATS_THRU_WRITE);
+                ios_dump_throughput_stats (list_head, this, logfp, IOS_STATS_THRU_WRITE);
         }
         return 0;
 }
@@ -926,8 +917,19 @@ ios_dump_args_init (struct ios_dump_args *args, ios_dump_type_t type,
         return ret;
 }
 
+static void
+ios_global_stats_clear (struct ios_global_stats *stats, struct timeval *now)
+{
+        GF_ASSERT (stats);
+        GF_ASSERT (now);
+
+        memset (stats, 0, sizeof (*stats));
+        stats->started_at = *now;
+}
+
 int
-io_stats_dump (xlator_t *this, struct ios_dump_args *args)
+io_stats_dump (xlator_t *this, struct ios_dump_args *args,
+               gf1_cli_info_op op, gf_boolean_t is_peek)
 {
         struct ios_conf         *conf = NULL;
         struct ios_global_stats  cumulative = {0, };
@@ -945,18 +947,32 @@ io_stats_dump (xlator_t *this, struct ios_dump_args *args)
         gettimeofday (&now, NULL);
         LOCK (&conf->lock);
         {
-                cumulative  = conf->cumulative;
-                incremental = conf->incremental;
+                if (op == GF_CLI_INFO_ALL ||
+                    op == GF_CLI_INFO_CUMULATIVE)
+                        cumulative  = conf->cumulative;
 
-                increment = conf->increment++;
+                if (op == GF_CLI_INFO_ALL ||
+                    op == GF_CLI_INFO_INCREMENTAL) {
+                        incremental = conf->incremental;
+                        increment = conf->increment;
 
-                memset (&conf->incremental, 0, sizeof (conf->incremental));
-                conf->incremental.started_at = now;
+                        if (!is_peek) {
+                                increment = conf->increment++;
+
+                                ios_global_stats_clear (&conf->incremental,
+                                                        &now);
+                        }
+                }
         }
         UNLOCK (&conf->lock);
 
-        io_stats_dump_global (this, &cumulative, &now, -1, args);
-        io_stats_dump_global (this, &incremental, &now, increment, args);
+        if (op == GF_CLI_INFO_ALL ||
+            op == GF_CLI_INFO_CUMULATIVE)
+                io_stats_dump_global (this, &cumulative, &now, -1, args);
+
+        if (op == GF_CLI_INFO_ALL ||
+            op == GF_CLI_INFO_INCREMENTAL)
+                io_stats_dump_global (this, &incremental, &now, increment, args);
 
         return 0;
 }
@@ -1078,35 +1094,44 @@ io_stats_dump_stats_to_dict (xlator_t *this, dict_t *resp,
         int                      ret = -1;
         ios_stats_thru_t         index = IOS_STATS_THRU_MAX;
         char                     timestr[256] = {0, };
+        char                    *dict_timestr = NULL;
 
         conf = this->private;
 
         switch (flags) {
-                case IOS_STATS_TYPE_OPEN: 
+                case IOS_STATS_TYPE_OPEN:
                         list_head = &conf->list[IOS_STATS_TYPE_OPEN];
                         LOCK (&conf->lock);
                         {
                                 ret = dict_set_uint64 (resp, "current-open",
                                                      conf->cumulative.nr_opens);
                                 if (ret)
-                                        goto out;
+                                        goto unlock;
                                 ret = dict_set_uint64 (resp, "max-open",
                                                        conf->cumulative.max_nr_opens);
 
                                 gf_time_fmt (timestr, sizeof timestr,
                                              conf->cumulative.max_openfd_time.tv_sec,
                                              gf_timefmt_FT);
-                                snprintf (timestr + strlen (timestr), sizeof timestr - strlen (timestr),
-                                          ".%"GF_PRI_SUSECONDS,
-                                          conf->cumulative.max_openfd_time.tv_usec);
-
-                                ret = dict_set_str (resp, "max-openfd-time",
-                                                    timestr);
+                                if (conf->cumulative.max_openfd_time.tv_sec)
+                                        snprintf (timestr + strlen (timestr), sizeof timestr - strlen (timestr),
+                                                  ".%"GF_PRI_SUSECONDS,
+                                                  conf->cumulative.max_openfd_time.tv_usec);
+
+                                dict_timestr = gf_strdup (timestr);
+                                if (!dict_timestr)
+                                        goto unlock;
+                                ret = dict_set_dynstr (resp, "max-openfd-time",
+                                                       dict_timestr);
                                 if (ret)
-                                        goto out;
+                                        goto unlock;
                         }
+        unlock:
                         UNLOCK (&conf->lock);
-
+                        /* Do not proceed if we came here because of some error
+                         * during the dict operation */
+                        if (ret)
+                                goto out;
                         break;
                 case IOS_STATS_TYPE_READ:
                         list_head = &conf->list[IOS_STATS_TYPE_READ];
@@ -1122,7 +1147,7 @@ io_stats_dump_stats_to_dict (xlator_t *this, dict_t *resp,
                         break;
                 case IOS_STATS_TYPE_READ_THROUGHPUT:
                         list_head = &conf->thru_list[IOS_STATS_THRU_READ];
-                        index = IOS_STATS_THRU_READ; 
+                        index = IOS_STATS_THRU_READ;
                         break;
                 case IOS_STATS_TYPE_WRITE_THROUGHPUT:
                         list_head = &conf->thru_list[IOS_STATS_THRU_WRITE];
@@ -1143,30 +1168,34 @@ io_stats_dump_stats_to_dict (xlator_t *this, dict_t *resp,
                         snprintf (key, 256, "%s-%d", "filename", cnt);
                         ret = dict_set_str (resp, key, entry->iosstat->filename);
                         if (ret)
-                                goto out;
+                                goto unlock_list_head;
                          snprintf (key, 256, "%s-%d", "value",cnt);
                          ret = dict_set_uint64 (resp, key, entry->value);
                          if (ret)
-                                 goto out;
+                                 goto unlock_list_head;
                          if (index != IOS_STATS_THRU_MAX) {
                                  snprintf (key, 256, "%s-%d", "time-sec", cnt);
-                                 ret = dict_set_int32 (resp, key, 
+                                 ret = dict_set_int32 (resp, key,
                                          entry->iosstat->thru_counters[index].time.tv_sec);
                                  if (ret)
-                                         goto out;
+                                         goto unlock_list_head;
                                  snprintf (key, 256, "%s-%d", "time-usec", cnt);
-                                 ret = dict_set_int32 (resp, key, 
+                                 ret = dict_set_int32 (resp, key,
                                          entry->iosstat->thru_counters[index].time.tv_usec);
                                  if (ret)
-                                         goto out;
+                                         goto unlock_list_head;
                          }
                          if (cnt == list_cnt)
                                  break;
 
                 }
         }
+unlock_list_head:
         UNLOCK (&list_head->lock);
-
+        /* ret is !=0 if some dict operation in the above critical region
+         * failed. */
+        if (ret)
+                goto out;
         ret = dict_set_int32 (resp, "members", cnt);
  out:
         return ret;
@@ -1243,7 +1272,7 @@ io_stats_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         struct   ios_stat *iosstat = NULL;
         struct ios_conf   *conf = NULL;
 
-	conf = this->private;
+        conf = this->private;
         path = frame->local;
         frame->local = NULL;
 
@@ -1267,6 +1296,16 @@ io_stats_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         ios_fd_ctx_set (fd, this, iosfd);
 
         ios_inode_ctx_get (fd->inode, this, &iosstat);
+        if (!iosstat) {
+                iosstat = GF_CALLOC (1, sizeof (*iosstat),
+                                     gf_io_stats_mt_ios_stat);
+                if (iosstat) {
+                        iosstat->filename = gf_strdup (path);
+                        uuid_copy (iosstat->gfid, fd->inode->gfid);
+                        LOCK_INIT (&iosstat->lock);
+                        ios_inode_ctx_set (fd->inode, this, iosstat);
+                }
+        }
 
         LOCK (&conf->lock);
         {
@@ -1349,7 +1388,7 @@ io_stats_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 ios_inode_ctx_get (inode, this, &iosstat);
                 if (iosstat) {
                         BUMP_STATS (iosstat, IOS_STATS_TYPE_WRITE);
-			BUMP_THROUGHPUT (iosstat, IOS_STATS_THRU_WRITE);
+                        BUMP_THROUGHPUT (iosstat, IOS_STATS_THRU_WRITE);
                         inode = NULL;
                         iosstat = NULL;
                 }
@@ -1519,8 +1558,7 @@ io_stats_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 unwind:
         /* local is assigned with path */
-        if (frame->local)
-                GF_FREE (frame->local);
+        GF_FREE (frame->local);
         frame->local = NULL;
         STACK_UNWIND_STRICT (mkdir, frame, op_ret, op_errno, inode, buf,
                              preparent, postparent, xdata);
@@ -1712,6 +1750,40 @@ io_stats_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 
 int
+io_stats_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+		       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+		       struct iatt *postbuf, dict_t *xdata)
+{
+	UPDATE_PROFILE_STATS(frame, FALLOCATE);
+	STACK_UNWIND_STRICT(fallocate, frame, op_ret, op_errno, prebuf, postbuf,
+			    xdata);
+	return 0;
+}
+
+
+int
+io_stats_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+		     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+		     struct iatt *postbuf, dict_t *xdata)
+{
+	UPDATE_PROFILE_STATS(frame, DISCARD);
+	STACK_UNWIND_STRICT(discard, frame, op_ret, op_errno, prebuf, postbuf,
+			    xdata);
+	return 0;
+}
+
+int
+io_stats_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
+{
+        UPDATE_PROFILE_STATS(frame, ZEROFILL);
+        STACK_UNWIND_STRICT(zerofill, frame, op_ret, op_errno, prebuf, postbuf,
+                            xdata);
+        return 0;
+}
+
+int
 io_stats_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
 {
@@ -2127,7 +2199,7 @@ io_stats_fsync (call_frame_t *frame, xlator_t *this,
 }
 
 
-void
+int
 conditional_dump (dict_t *dict, char *key, data_t *value, void *data)
 {
         struct {
@@ -2151,20 +2223,20 @@ conditional_dump (dict_t *dict, char *key, data_t *value, void *data)
 
                 if (!strncmp (filename, "", 1)) {
                         gf_log (this->name, GF_LOG_ERROR, "No filename given");
-                        return;
+                        return -1;
                 }
                 logfp = fopen (filename, "w+");
-                GF_ASSERT (logfp);
                 if (!logfp) {
                         gf_log (this->name, GF_LOG_ERROR, "failed to open %s "
                                 "for writing", filename);
-                        return;
-                }
+                        return -1;
+                    }
                 (void) ios_dump_args_init (&args, IOS_DUMP_TYPE_FILE,
                                            logfp);
-                io_stats_dump (this, &args);
+                io_stats_dump (this, &args, GF_CLI_INFO_ALL, _gf_false);
                 fclose (logfp);
         }
+        return 0;
 }
 
 
@@ -2380,6 +2452,45 @@ io_stats_fstat (call_frame_t *frame, xlator_t *this,
 
 
 int
+io_stats_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+		   off_t offset, size_t len, dict_t *xdata)
+{
+	START_FOP_LATENCY(frame);
+
+	STACK_WIND(frame, io_stats_fallocate_cbk, FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fallocate, fd, mode, offset, len,
+		   xdata);
+
+	return 0;
+}
+
+
+int
+io_stats_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+		 size_t len, dict_t *xdata)
+{
+	START_FOP_LATENCY(frame);
+
+	STACK_WIND(frame, io_stats_discard_cbk, FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->discard, fd, offset, len, xdata);
+
+	return 0;
+}
+
+int
+io_stats_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                 off_t len, dict_t *xdata)
+{
+        START_FOP_LATENCY(frame);
+
+        STACK_WIND(frame, io_stats_zerofill_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->zerofill, fd, offset, len, xdata);
+
+        return 0;
+}
+
+
+int
 io_stats_lk (call_frame_t *frame, xlator_t *this,
              fd_t *fd, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
 {
@@ -2405,7 +2516,7 @@ io_stats_release (xlator_t *this, fd_t *fd)
 
         LOCK (&conf->lock);
         {
-		conf->cumulative.nr_opens--;
+                conf->cumulative.nr_opens--;
         }
         UNLOCK (&conf->lock);
 
@@ -2413,8 +2524,7 @@ io_stats_release (xlator_t *this, fd_t *fd)
         if (iosfd) {
                 io_stats_dump_fd (this, iosfd);
 
-                if (iosfd->filename)
-                        GF_FREE (iosfd->filename);
+                GF_FREE (iosfd->filename);
                 GF_FREE (iosfd);
         }
 
@@ -2439,6 +2549,116 @@ io_stats_forget (xlator_t *this, inode_t *inode)
         return 0;
 }
 
+static int
+ios_init_top_stats (struct ios_conf *conf)
+{
+        int     i = 0;
+
+        GF_ASSERT (conf);
+
+        for (i = 0; i <IOS_STATS_TYPE_MAX; i++) {
+                conf->list[i].iosstats = GF_CALLOC (1,
+                                         sizeof(*conf->list[i].iosstats),
+                                         gf_io_stats_mt_ios_stat);
+
+                if (!conf->list[i].iosstats)
+                        return -1;
+
+                INIT_LIST_HEAD(&conf->list[i].iosstats->list);
+                LOCK_INIT (&conf->list[i].lock);
+        }
+
+        for (i = 0; i < IOS_STATS_THRU_MAX; i ++) {
+                conf->thru_list[i].iosstats = GF_CALLOC (1,
+                                 sizeof (*conf->thru_list[i].iosstats),
+                                 gf_io_stats_mt_ios_stat);
+
+                if (!conf->thru_list[i].iosstats)
+                        return -1;
+
+                INIT_LIST_HEAD(&conf->thru_list[i].iosstats->list);
+                LOCK_INIT (&conf->thru_list[i].lock);
+        }
+
+        return 0;
+}
+
+static void
+ios_destroy_top_stats (struct ios_conf *conf)
+{
+        int                     i = 0;
+        struct ios_stat_head    *list_head = NULL;
+        struct ios_stat_list    *entry     = NULL;
+        struct ios_stat_list    *tmp       = NULL;
+        struct ios_stat_list    *list      = NULL;
+        struct ios_stat         *stat      = NULL;
+
+        GF_ASSERT (conf);
+
+        LOCK (&conf->lock);
+
+        conf->cumulative.nr_opens = 0;
+        conf->cumulative.max_nr_opens = 0;
+        conf->cumulative.max_openfd_time.tv_sec = 0;
+        conf->cumulative.max_openfd_time.tv_usec = 0;
+
+        for (i = 0; i < IOS_STATS_TYPE_MAX; i++) {
+                list_head = &conf->list[i];
+                if (!list_head)
+                        continue;
+                list_for_each_entry_safe (entry, tmp,
+                                          &list_head->iosstats->list, list) {
+                        list = entry;
+                        stat = list->iosstat;
+                        ios_stat_unref (stat);
+                        list_del (&list->list);
+                        GF_FREE (list);
+                        list_head->members--;
+                }
+        }
+
+        for (i = 0; i < IOS_STATS_THRU_MAX; i++) {
+                list_head = &conf->thru_list[i];
+                if (!list_head)
+                        continue;
+                list_for_each_entry_safe (entry, tmp,
+                                          &list_head->iosstats->list, list) {
+                        list = entry;
+                        stat = list->iosstat;
+                        ios_stat_unref (stat);
+                        list_del (&list->list);
+                        GF_FREE (list);
+                        list_head->members--;
+                }
+        }
+
+        UNLOCK (&conf->lock);
+
+        return;
+}
+
+static int
+io_stats_clear (struct ios_conf *conf)
+{
+        struct timeval      now;
+        int                 ret = -1;
+
+        GF_ASSERT (conf);
+
+        if (!gettimeofday (&now, NULL))
+        {
+            LOCK (&conf->lock);
+            {
+                    ios_global_stats_clear (&conf->cumulative, &now);
+                    ios_global_stats_clear (&conf->incremental, &now);
+                    conf->increment = 0;
+            }
+            UNLOCK (&conf->lock);
+            ret = 0;
+        }
+
+        return ret;
+}
 
 int
 reconfigure (xlator_t *this, dict_t *options)
@@ -2506,7 +2726,6 @@ int
 init (xlator_t *this)
 {
         struct ios_conf    *conf = NULL;
-        int                 i = 0;
         char               *sys_log_str = NULL;
         int                 sys_log_level = -1;
         char               *log_str = NULL;
@@ -2543,35 +2762,9 @@ init (xlator_t *this)
         gettimeofday (&conf->cumulative.started_at, NULL);
         gettimeofday (&conf->incremental.started_at, NULL);
 
-        for (i = 0; i <IOS_STATS_TYPE_MAX; i++) {
-                conf->list[i].iosstats = GF_CALLOC (1,
-                                         sizeof(*conf->list[i].iosstats),
-                                         gf_io_stats_mt_ios_stat);
-
-                if (!conf->list[i].iosstats) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                               "Out of memory");
-                        return -1;
-                }
-
-                INIT_LIST_HEAD(&conf->list[i].iosstats->list);
-                LOCK_INIT (&conf->list[i].lock);
-        }
-
-	for (i = 0; i < IOS_STATS_THRU_MAX; i ++) {
-		conf->thru_list[i].iosstats = GF_CALLOC (1,
-		                 sizeof (*conf->thru_list[i].iosstats),
-				 gf_io_stats_mt_ios_stat);
-
-	        if (!conf->thru_list[i].iosstats) {
-        	        gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory");
-                	return -1;
-        	}
-
-		INIT_LIST_HEAD(&conf->thru_list[i].iosstats->list);
-		LOCK_INIT (&conf->thru_list[i].lock);
-        }
+        ret = ios_init_top_stats (conf);
+        if (ret)
+                return -1;
 
         GF_OPTION_INIT ("dump-fd-stats", conf->dump_fd_stats, bool, out);
 
@@ -2603,12 +2796,6 @@ void
 fini (xlator_t *this)
 {
         struct ios_conf *conf = NULL;
-        struct ios_stat_head *list_head = NULL;
-        struct ios_stat_list *entry     = NULL;
-        struct ios_stat_list *tmp       = NULL;
-        struct ios_stat_list *list      = NULL;
-        struct ios_stat      *stat      = NULL;
-        int    i                        = 0;
 
         if (!this)
                 return;
@@ -2619,45 +2806,15 @@ fini (xlator_t *this)
                 return;
         this->private = NULL;
 
-        for (i = 0; i < IOS_STATS_TYPE_MAX; i++) {
-                list_head = &conf->list[i];
-                if (!list_head)
-                        continue;
-                list_for_each_entry_safe (entry, tmp,
-                                          &list_head->iosstats->list, list) {
-                        list = entry;
-                        stat = list->iosstat;
-                        ios_stat_unref (stat);
-                        list_del (&list->list);
-                        if (list)
-                                GF_FREE (list);
-                }
-        }
-
-        for (i = 0; i < IOS_STATS_THRU_MAX; i++) {
-                list_head = &conf->thru_list[i];
-                if (!list_head)
-                        continue;
-                list_for_each_entry_safe (entry, tmp,
-                                          &list_head->iosstats->list, list) {
-                        list = entry;
-                        stat = list->iosstat;
-                        ios_stat_unref (stat);
-                        list_del (&list->list);
-                        if (list)
-                                GF_FREE (list);
-                }
-        }
+        ios_destroy_top_stats (conf);
 
-        if (conf)
-                GF_FREE(conf);
+        GF_FREE(conf);
 
         gf_log (this->name, GF_LOG_INFO,
                 "io-stats translator unloaded");
         return;
 }
 
-
 int
 notify (xlator_t *this, int32_t event, void *data, ...)
 {
@@ -2665,10 +2822,11 @@ notify (xlator_t *this, int32_t event, void *data, ...)
         struct ios_dump_args args = {0};
         dict_t       *output = NULL;
         dict_t       *dict = NULL;
-        int32_t       top_op = 0;
+        int32_t       op = 0;
         int32_t       list_cnt = 0;
         double        throughput = 0;
         double        time = 0;
+        gf_boolean_t  is_peek = _gf_false;
         va_list ap;
 
         dict = data;
@@ -2677,15 +2835,37 @@ notify (xlator_t *this, int32_t event, void *data, ...)
         va_end (ap);
         switch (event) {
         case GF_EVENT_TRANSLATOR_INFO:
-                ret = dict_get_int32 (dict, "top-op", &top_op);
+                ret = dict_get_str_boolean (dict, "clear-stats", _gf_false);
+                if (ret) {
+                        ret = dict_set_int32 (output, "top-op", op);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to set top-op in dict");
+                                goto out;
+                        }
+                        ios_destroy_top_stats (this->private);
+                        ret = ios_init_top_stats (this->private);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to reset top stats");
+                        ret = dict_set_int32 (output, "stats-cleared",
+                                              ret ? 0 : 1);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to set stats-cleared"
+                                        " in dict");
+                        goto out;
+                }
+
+                ret = dict_get_int32 (dict, "top-op", &op);
                 if (!ret) {
                         ret = dict_get_int32 (dict, "list-cnt", &list_cnt);
-                        if (top_op > IOS_STATS_TYPE_NONE &&
-                            top_op < IOS_STATS_TYPE_MAX)
+                        if (op > IOS_STATS_TYPE_NONE &&
+                            op < IOS_STATS_TYPE_MAX)
                                 ret = io_stats_dump_stats_to_dict (this, output,
-                                                             top_op, list_cnt);
-                        if (top_op == IOS_STATS_TYPE_READ_THROUGHPUT ||
-                                top_op == IOS_STATS_TYPE_WRITE_THROUGHPUT) {
+                                                             op, list_cnt);
+                        if (op == IOS_STATS_TYPE_READ_THROUGHPUT ||
+                                op == IOS_STATS_TYPE_WRITE_THROUGHPUT) {
                                 ret = dict_get_double (dict, "throughput",
                                                         &throughput);
                                 if (!ret) {
@@ -2706,9 +2886,41 @@ notify (xlator_t *this, int32_t event, void *data, ...)
 
                         }
                 } else {
-                        (void) ios_dump_args_init (&args, IOS_DUMP_TYPE_DICT,
-                                           output);
-                        ret = io_stats_dump (this, &args);
+                        ret = dict_get_int32 (dict, "info-op", &op);
+                        if (ret || op < GF_CLI_INFO_ALL ||
+                            GF_CLI_INFO_CLEAR < op)
+                            op = GF_CLI_INFO_ALL;
+
+                        ret = dict_set_int32 (output, "info-op", op);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to set info-op in dict");
+                                goto out;
+                        }
+
+                        if (GF_CLI_INFO_CLEAR == op) {
+                                ret = io_stats_clear (this->private);
+                                if (ret)
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "Failed to clear info stats");
+
+                                ret = dict_set_int32 (output, "stats-cleared",
+                                              ret ? 0 : 1);
+                                if (ret)
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "Failed to set stats-cleared"
+                                                " in dict");
+                        }
+                        else {
+                                ret = dict_get_str_boolean (dict, "peek",
+                                                            _gf_false);
+                                if (-1 != ret)
+                                        is_peek = ret;
+
+                                (void) ios_dump_args_init (&args,
+                                                IOS_DUMP_TYPE_DICT, output);
+                                ret = io_stats_dump (this, &args, op, is_peek);
+                        }
                 }
                 break;
         default:
@@ -2760,6 +2972,9 @@ struct xlator_fops fops = {
         .fxattrop    = io_stats_fxattrop,
         .setattr     = io_stats_setattr,
         .fsetattr    = io_stats_fsetattr,
+	.fallocate   = io_stats_fallocate,
+	.discard     = io_stats_discard,
+        .zerofill    = io_stats_zerofill,
 };
 
 struct xlator_cbks cbks = {
@@ -2803,7 +3018,7 @@ struct volume_options options[] = {
           .type = GF_OPTION_TYPE_STR,
           .default_value = "CRITICAL",
           .description = "Gluster's syslog log-level",
-          .value = { "WARNING", "ERROR", "CRITICAL"}
+          .value = { "WARNING", "ERROR", "INFO", "CRITICAL"}
         },
         { .key = {"brick-log-level"},
           .type = GF_OPTION_TYPE_STR,
diff --git a/xlators/debug/trace/src/Makefile.am b/xlators/debug/trace/src/Makefile.am
index 0f1679a04..7b2597b4d 100644
--- a/xlators/debug/trace/src/Makefile.am
+++ b/xlators/debug/trace/src/Makefile.am
@@ -2,13 +2,15 @@
 xlator_LTLIBRARIES = trace.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/debug
 
-trace_la_LDFLAGS = -module -avoidversion
+trace_la_LDFLAGS = -module -avoid-version
 
 trace_la_SOURCES = trace.c
 trace_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)\
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+noinst_HEADERS = trace.h trace-mem-types.h
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES = 
 
diff --git a/xlators/debug/trace/src/trace-mem-types.h b/xlators/debug/trace/src/trace-mem-types.h
new file mode 100644
index 000000000..9fa7d97c2
--- /dev/null
+++ b/xlators/debug/trace/src/trace-mem-types.h
@@ -0,0 +1,21 @@
+/*
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef __TRACE_MEM_TYPES_H__
+#define __TRACE_MEM_TYPES_H__
+
+#include "mem-types.h"
+
+enum gf_trace_mem_types_ {
+        gf_trace_mt_trace_conf_t = gf_common_mt_end + 1,
+        gf_trace_mt_end
+};
+#endif
diff --git a/xlators/debug/trace/src/trace.c b/xlators/debug/trace/src/trace.c
index a1136a2e5..c9d839356 100644
--- a/xlators/debug/trace/src/trace.c
+++ b/xlators/debug/trace/src/trace.c
@@ -1,26 +1,15 @@
 /*
-  Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  General Public License for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
+#include "trace.h"
+#include "trace-mem-types.h"
 
 /**
  * xlators/debug/trace :
@@ -29,897 +18,977 @@
  *    Very helpful translator for debugging.
  */
 
-#include <time.h>
-#include <errno.h>
-#include "glusterfs.h"
-#include "xlator.h"
-#include "common-utils.h"
-
-
-struct {
-        char *name;
-        int enabled;
-} trace_fop_names[GF_FOP_MAXVALUE];
-
-int trace_log_level = GF_LOG_INFO;
-
-static char *
-trace_stat_to_str (struct iatt *buf)
+int
+dump_history_trace (circular_buffer_t *cb, void *data)
 {
-        char     *statstr           = NULL;
-        char      atime_buf[64]     = {0,};
-        char      mtime_buf[64]     = {0,};
-        char      ctime_buf[64]     = {0,};
-        int       asprint_ret_value = 0;
+        char    *string      = NULL;
+        struct  tm      *tm  = NULL;
+        char    timestr[256] = {0,};
 
-        if (!buf) {
-                statstr = NULL;
-                goto out;
-        }
+        string = (char *)cb->data;
+        tm    = localtime (&cb->tv.tv_sec);
 
-        gf_time_fmt (atime_buf, sizeof atime_buf, buf->ia_atime, gf_timefmt_bdT);
-        gf_time_fmt (mtime_buf, sizeof mtime_buf, buf->ia_mtime, gf_timefmt_bdT);
-        gf_time_fmt (ctime_buf, sizeof ctime_buf, buf->ia_ctime, gf_timefmt_bdT);
-        asprint_ret_value = gf_asprintf (&statstr,
-                                         "gfid=%s ino=%"PRIu64", mode=%o, "
-                                         "nlink=%"GF_PRI_NLINK", uid=%u, "
-                                         "gid=%u, size=%"PRIu64", "
-                                         "blocks=%"PRIu64", atime=%s, "
-                                         "mtime=%s, ctime=%s",
-                                         uuid_utoa (buf->ia_gfid), buf->ia_ino,
-                                         st_mode_from_ia (buf->ia_prot,
-                                                          buf->ia_type),
-                                         buf->ia_nlink, buf->ia_uid,
-                                         buf->ia_gid, buf->ia_size,
-                                         buf->ia_blocks, atime_buf,
-                                         mtime_buf, ctime_buf);
+        /* Since we are continuing with adding entries to the buffer even when
+           gettimeofday () fails, it's safe to check tm and then dump the time
+           at which the entry was added to the buffer */
+        if (tm) {
+                strftime (timestr, 256, "%Y-%m-%d %H:%M:%S", tm);
+                snprintf (timestr + strlen (timestr), 256 - strlen (timestr),
+                          ".%"GF_PRI_SUSECONDS, cb->tv.tv_usec);
+                gf_proc_dump_write ("TIME", "%s", timestr);
+        }
 
-        if (asprint_ret_value < 0)
-                statstr = NULL;
+        gf_proc_dump_write ("FOP", "%s\n", string);
 
-out:
-        return statstr;
+        return 0;
 }
 
-
 int
 trace_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno, fd_t *fd,
                   inode_t *inode, struct iatt *buf,
-                  struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+                  struct iatt *preparent, struct iatt *postparent,
+                  dict_t *xdata)
 {
-        char  *statstr = NULL;
-        char  *preparentstr = NULL;
-        char  *postparentstr = NULL;
+        char          statstr[4096]       = {0, };
+        char          preparentstr[4096]  = {0, };
+        char          postparentstr[4096] = {0, };
+        trace_conf_t  *conf               = NULL;
+
+        conf = this->private;
 
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_CREATE].enabled) {
+                char  string[4096] = {0,};
                 if (op_ret >= 0) {
-                        statstr = trace_stat_to_str (buf);
-                        preparentstr = trace_stat_to_str (preparent);
-                        postparentstr = trace_stat_to_str (postparent);
-
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s (op_ret=%d, fd=%p"
-                                "*stbuf {%s}, *preparent {%s}, *postparent = "
-                                "{%s})", frame->root->unique,
-                                uuid_utoa (inode->gfid), op_ret, fd,
-                                statstr, preparentstr, postparentstr);
-
-                        if (statstr)
-                                GF_FREE (statstr);
-                        if (preparentstr)
-                                GF_FREE (preparentstr);
-                        if (postparentstr)
-                                GF_FREE (postparentstr);
+                        trace_stat_to_str (buf, statstr);
+                        trace_stat_to_str (preparent, preparentstr);
+                        trace_stat_to_str (postparent, postparentstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s (op_ret=%d, fd=%p"
+                                  "*stbuf {%s}, *preparent {%s}, "
+                                  "*postparent = {%s})",
+                                  frame->root->unique,
+                                  uuid_utoa (inode->gfid), op_ret, fd,
+                                  statstr, preparentstr, postparentstr);
 
                         /* for 'release' log */
                         fd_ctx_set (fd, this, 0);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": (op_ret=%d, op_errno=%d)",
-                                frame->root->unique, op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, op_errno=%d)",
+                                  frame->root->unique, op_ret,
+                                  op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (create, frame, op_ret, op_errno, fd, inode, buf,
-                             preparent, postparent, xdata);
+out:
+        TRACE_STACK_UNWIND (create, frame, op_ret, op_errno, fd, inode, buf,
+                            preparent, postparent, xdata);
         return 0;
 }
 
-
 int
 trace_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
+        trace_conf_t      *conf = NULL;
 
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_OPEN].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s op_ret=%d, op_errno=%d, *fd=%p",
-                        frame->root->unique, uuid_utoa (frame->local), op_ret, op_errno, fd);
+                char     string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d, "
+                          "*fd=%p", frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno,
+                          fd);
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         /* for 'release' log */
         if (op_ret >= 0)
                 fd_ctx_set (fd, this, 0);
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, xdata);
+        TRACE_STACK_UNWIND (open, frame, op_ret, op_errno, fd, xdata);
         return 0;
 }
 
-
 int
 trace_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
+                int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                dict_t *xdata)
 {
-        char *statstr = NULL;
+        char          statstr[4096] = {0, };
+        trace_conf_t  *conf         = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_STAT].enabled) {
-                if (op_ret >= 0) {
-                        statstr = trace_stat_to_str (buf);
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d buf=%s",
-                                frame->root->unique, uuid_utoa (frame->local),
-                                op_ret, statstr);
-
-                        if (statstr)
-                                GF_FREE (statstr);
+                char string[4096] = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (buf, statstr);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d buf=%s",
+                                  frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  statstr);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d, op_errno=%d)",
-                                frame->root->unique, uuid_utoa (frame->local), op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d)",
+                                  frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (stat, frame, op_ret, op_errno, buf, xdata);
+out:
+        TRACE_STACK_UNWIND (stat, frame, op_ret, op_errno, buf, xdata);
         return 0;
 }
 
-
 int
 trace_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno, struct iovec *vector,
-                 int32_t count, struct iatt *buf, struct iobref *iobref, dict_t *xdata)
+                 int32_t count, struct iatt *buf, struct iobref *iobref,
+                 dict_t *xdata)
 {
-        char  *statstr = NULL;
+        char          statstr[4096] = {0, };
+        trace_conf_t  *conf         = NULL;
+
+        conf = this->private;
 
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_READ].enabled) {
+                char  string[4096] = {0,};
                 if (op_ret >= 0) {
-                        statstr = trace_stat_to_str (buf);
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d buf=%s",
-                                frame->root->unique, uuid_utoa (frame->local),
-                                op_ret, statstr);
-
-                        if (statstr)
-                                GF_FREE (statstr);
+                        trace_stat_to_str (buf, statstr);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d buf=%s",
+                                  frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  statstr);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d, op_errno=%d)",
-                                frame->root->unique, uuid_utoa (frame->local),
-                                op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d)",
+                                  frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno, vector, count,
-                             buf, iobref, xdata);
+out:
+        TRACE_STACK_UNWIND (readv, frame, op_ret, op_errno, vector, count,
+                            buf, iobref, xdata);
         return 0;
 }
 
-
 int
 trace_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno,
                   struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
 {
-        char  *preopstr = NULL;
-        char  *postopstr = NULL;
+        char         preopstr[4096]  = {0, };
+        char         postopstr[4096] = {0, };
+        trace_conf_t *conf           = NULL;
 
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_WRITE].enabled) {
+                char  string[4096] = {0,};
                 if (op_ret >= 0) {
-                        preopstr = trace_stat_to_str (prebuf);
-                        postopstr = trace_stat_to_str (postbuf);
-
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": (op_ret=%d, *prebuf = {%s}, "
-                                "*postbuf = {%s})",
-                                frame->root->unique, op_ret,
-                                preopstr, postopstr);
-
-                        if (preopstr)
-                                GF_FREE (preopstr);
-
-                        if (postopstr)
-                                GF_FREE (postopstr);
+                        trace_stat_to_str (prebuf, preopstr);
+                        trace_stat_to_str (postbuf, postopstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, "
+                                  "*prebuf = {%s}, *postbuf = {%s})",
+                                  frame->root->unique, op_ret,
+                                  preopstr, postopstr);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                                frame->root->unique, uuid_utoa (frame->local),
-                                op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d", frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf, xdata);
+out:
+        TRACE_STACK_UNWIND (writev, frame, op_ret, op_errno, prebuf, postbuf,
+                            xdata);
         return 0;
 }
 
-
-
 int
 trace_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, gf_dirent_t *buf, dict_t *xdata)
+                   int32_t op_ret, int32_t op_errno, gf_dirent_t *buf,
+                   dict_t *xdata)
 {
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_READDIR].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64" : gfid=%s op_ret=%d, op_errno=%d",
-                        frame->root->unique, uuid_utoa (frame->local), op_ret, op_errno);
-        }
+                char    string[4096]  = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64" : gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique, uuid_utoa (frame->local),
+                          op_ret, op_errno);
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, buf, xdata);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (readdir, frame, op_ret, op_errno, buf, xdata);
 
         return 0;
 }
 
-
 int
 trace_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, gf_dirent_t *buf, dict_t *xdata)
+                    int32_t op_ret, int32_t op_errno, gf_dirent_t *buf,
+                    dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_READDIRP].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64" : gfid=%s op_ret=%d, op_errno=%d",
-                        frame->root->unique, uuid_utoa (frame->local), op_ret, op_errno);
-        }
+                char        string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64" : gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique, uuid_utoa (frame->local),
+                          op_ret, op_errno);
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, buf, xdata);
+                LOG_ELEMENT (conf, string);
+        }
 
+out:
+        TRACE_STACK_UNWIND (readdirp, frame, op_ret, op_errno, buf, xdata);
         return 0;
 }
 
-
 int
 trace_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno,
                  struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
 {
-        char  *preopstr = NULL;
-        char  *postopstr = NULL;
-
-        if (trace_fop_names[GF_FOP_FSYNC].enabled) {
-                if (op_ret >= 0) {
-                        preopstr = trace_stat_to_str (prebuf);
-                        postopstr = trace_stat_to_str (postbuf);
-
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": (op_ret=%d, *prebuf = {%s}, "
-                                "*postbuf = {%s}",
-                                frame->root->unique, op_ret,
-                                preopstr, postopstr);
+        char          preopstr[4096]  = {0, };
+        char          postopstr[4096] = {0, };
+        trace_conf_t  *conf           = NULL;
 
-                        if (preopstr)
-                                GF_FREE (preopstr);
+        conf = this->private;
 
-                        if (postopstr)
-                                GF_FREE (postopstr);
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_FSYNC].enabled) {
+                char  string[4096] = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (prebuf, preopstr);
+                        trace_stat_to_str (postbuf, postopstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, "
+                                  "*prebuf = {%s}, *postbuf = {%s}",
+                                  frame->root->unique, op_ret,
+                                  preopstr, postopstr);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                                frame->root->unique, uuid_utoa (frame->local),
-                                op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d", frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
+
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (fsync, frame, op_ret, op_errno, prebuf, postbuf, xdata);
+out:
+        TRACE_STACK_UNWIND (fsync, frame, op_ret, op_errno, prebuf, postbuf,
+                            xdata);
 
         return 0;
 }
 
-
 int
 trace_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno,
                    struct iatt *statpre, struct iatt *statpost, dict_t *xdata)
 {
-        char  *preopstr = NULL;
-        char  *postopstr = NULL;
-
-        if (trace_fop_names[GF_FOP_SETATTR].enabled) {
-                if (op_ret >= 0) {
-                        preopstr = trace_stat_to_str (statpre);
-                        postopstr = trace_stat_to_str (statpost);
-
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": (op_ret=%d, *prebuf = {%s}, "
-                                "*postbuf = {%s})",
-                                frame->root->unique, op_ret,
-                                preopstr, postopstr);
+        char          preopstr[4096]  = {0, };
+        char          postopstr[4096] = {0, };
+        trace_conf_t  *conf           = NULL;
 
-                        if (preopstr)
-                                GF_FREE (preopstr);
+        conf = this->private;
 
-                        if (postopstr)
-                                GF_FREE (postopstr);
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_SETATTR].enabled) {
+                char  string[4096]  = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (statpre, preopstr);
+                        trace_stat_to_str (statpost, postopstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, "
+                                  "*prebuf = {%s}, *postbuf = {%s})",
+                                  frame->root->unique, op_ret,
+                                  preopstr, postopstr);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d, op_errno=%d)",
-                                frame->root->unique, uuid_utoa (frame->local), op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d)", frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (setattr, frame, op_ret, op_errno, statpre, statpost, xdata);
+out:
+        TRACE_STACK_UNWIND (setattr, frame, op_ret, op_errno, statpre,
+                            statpost, xdata);
         return 0;
 }
 
-
 int
 trace_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno,
                     struct iatt *statpre, struct iatt *statpost, dict_t *xdata)
 {
-        char  *preopstr = NULL;
-        char  *postopstr = NULL;
+        char          preopstr[4096]  = {0, };
+        char          postopstr[4096] = {0, };
+        trace_conf_t  *conf           = NULL;
 
-        if (trace_fop_names[GF_FOP_FSETATTR].enabled) {
-                if (op_ret >= 0) {
-                        preopstr = trace_stat_to_str (statpre);
-                        postopstr = trace_stat_to_str (statpost);
+        conf = this->private;
 
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": (op_ret=%d, *prebuf = {%s}, "
-                                "*postbuf = {%s})",
-                                frame->root->unique, op_ret,
-                                preopstr, postopstr);
-
-                        if (preopstr)
-                                GF_FREE (preopstr);
-
-                        if (postopstr)
-                                GF_FREE (postopstr);
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_FSETATTR].enabled) {
+                char  string[4096] = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (statpre, preopstr);
+                        trace_stat_to_str (statpost, postopstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, "
+                                  "*prebuf = {%s}, *postbuf = {%s})",
+                                  frame->root->unique, op_ret,
+                                  preopstr, postopstr);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d, op_errno=%d)",
-                                frame->root->unique, uuid_utoa (frame->local), op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, op_errno=%d)",
+                                  frame->root->unique, uuid_utoa (frame->local),
+                                  op_ret, op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (fsetattr, frame, op_ret, op_errno,
-                             statpre, statpost, xdata);
+out:
+        TRACE_STACK_UNWIND (fsetattr, frame, op_ret, op_errno,
+                            statpre, statpost, xdata);
         return 0;
 }
 
-
 int
 trace_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno,
-                  struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+                  struct iatt *preparent, struct iatt *postparent,
+                  dict_t *xdata)
 {
-        char *preparentstr = NULL;
-        char *postparentstr = NULL;
-
-        if (trace_fop_names[GF_FOP_UNLINK].enabled) {
-                if (op_ret >= 0) {
-                        preparentstr = trace_stat_to_str (preparent);
-                        postparentstr = trace_stat_to_str (postparent);
-
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d, *preparent = {%s}, "
-                                "*postparent = {%s})",
-                                frame->root->unique, uuid_utoa (frame->local), op_ret, preparentstr,
-                                postparentstr);
+        char          preparentstr[4096]  = {0, };
+        char          postparentstr[4096] = {0, };
+        trace_conf_t  *conf               = NULL;
 
-                        if (preparentstr)
-                                GF_FREE (preparentstr);
+        conf = this->private;
 
-                        if (postparentstr)
-                                GF_FREE (postparentstr);
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_UNLINK].enabled) {
+                char string[4096] = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (preparent, preparentstr);
+                        trace_stat_to_str (postparent, postparentstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  " *preparent = {%s}, "
+                                  "*postparent = {%s})",
+                                  frame->root->unique,
+                                  uuid_utoa (frame->local),
+                                  op_ret, preparentstr,
+                                  postparentstr);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d, op_errno=%d)",
-                                frame->root->unique, uuid_utoa (frame->local), op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d)",
+                                  frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (unlink, frame, op_ret, op_errno,
-                             preparent, postparent, xdata);
+out:
+        TRACE_STACK_UNWIND (unlink, frame, op_ret, op_errno,
+                            preparent, postparent, xdata);
         return 0;
 }
 
-
 int
 trace_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno, struct iatt *buf,
                   struct iatt *preoldparent, struct iatt *postoldparent,
-                  struct iatt *prenewparent, struct iatt *postnewparent, dict_t *xdata)
+                  struct iatt *prenewparent, struct iatt *postnewparent,
+                  dict_t *xdata)
 {
-        char  *statstr = NULL;
-        char  *preoldparentstr = NULL;
-        char  *postoldparentstr = NULL;
-        char  *prenewparentstr = NULL;
-        char  *postnewparentstr = NULL;
-
-        if (trace_fop_names[GF_FOP_RENAME].enabled) {
-                if (op_ret >= 0) {
-                        statstr = trace_stat_to_str (buf);
-                        preoldparentstr = trace_stat_to_str (preoldparent);
-                        postoldparentstr = trace_stat_to_str (postoldparent);
-
-                        prenewparentstr = trace_stat_to_str (prenewparent);
-                        postnewparentstr = trace_stat_to_str (postnewparent);
-
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": (op_ret=%d, *stbuf = {%s}, "
-                                "*preoldparent = {%s}, *postoldparent = {%s}"
-                                " *prenewparent = {%s}, *postnewparent = {%s})",
-                                frame->root->unique, op_ret, statstr,
-                                preoldparentstr, postoldparentstr,
-                                prenewparentstr, postnewparentstr);
-
-                        if (statstr)
-                                GF_FREE (statstr);
-
-                        if (preoldparentstr)
-                                GF_FREE (preoldparentstr);
-
-                        if (postoldparentstr)
-                                GF_FREE (postoldparentstr);
+        char           statstr[4096]          = {0, };
+        char           preoldparentstr[4096]  = {0, };
+        char           postoldparentstr[4096] = {0, };
+        char           prenewparentstr[4096]  = {0, };
+        char           postnewparentstr[4096] = {0, };
+        trace_conf_t   *conf                  = NULL;
 
-                        if (prenewparentstr)
-                                GF_FREE (prenewparentstr);
+        conf = this->private;
 
-                        if (postnewparentstr)
-                                GF_FREE (postnewparentstr);
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_RENAME].enabled) {
+                char  string[4096] = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (buf, statstr);
+                        trace_stat_to_str (preoldparent, preoldparentstr);
+                        trace_stat_to_str (postoldparent, postoldparentstr);
+                        trace_stat_to_str (prenewparent, prenewparentstr);
+                        trace_stat_to_str (postnewparent, postnewparentstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, "
+                                  "*stbuf = {%s}, *preoldparent = {%s},"
+                                  " *postoldparent = {%s}"
+                                  " *prenewparent = {%s}, "
+                                  "*postnewparent = {%s})",
+                                  frame->root->unique, op_ret, statstr,
+                                  preoldparentstr, postoldparentstr,
+                                  prenewparentstr, postnewparentstr);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                                frame->root->unique, uuid_utoa (frame->local),
-                                op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d", frame->root->unique,
+                                  uuid_utoa (frame->local),
+                                  op_ret, op_errno);
+
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (rename, frame, op_ret, op_errno, buf,
-                             preoldparent, postoldparent,
-                             prenewparent, postnewparent, xdata);
+out:
+        TRACE_STACK_UNWIND (rename, frame, op_ret, op_errno, buf,
+                            preoldparent, postoldparent,
+                            prenewparent, postnewparent, xdata);
         return 0;
 }
 
-
 int
 trace_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno,
                     const char *buf, struct iatt *stbuf, dict_t *xdata)
 {
-        char *statstr = NULL;
+        char          statstr[4096] = {0, };
+        trace_conf_t  *conf         = NULL;
 
-        if (trace_fop_names[GF_FOP_READLINK].enabled) {
+        conf = this->private;
 
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_READLINK].enabled) {
+                char string[4096] = {0,};
                 if (op_ret == 0) {
-                        statstr = trace_stat_to_str (stbuf);
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": (op_ret=%d, op_errno=%d, buf=%s, "
-                                "stbuf = { %s })",
-                                frame->root->unique, op_ret, op_errno, buf,
-                                statstr);
-                } else
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                                frame->root->unique, uuid_utoa (frame->local),
-                                op_ret, op_errno);
+                        trace_stat_to_str (stbuf, statstr);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, op_errno=%d,"
+                                  "buf=%s, stbuf = { %s })",
+                                  frame->root->unique, op_ret, op_errno,
+                                  buf, statstr);
+                } else {
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d",
+                                  frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
+                }
 
-                if (statstr)
-                        GF_FREE (statstr);
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (readlink, frame, op_ret, op_errno, buf, stbuf, xdata);
+out:
+        TRACE_STACK_UNWIND (readlink, frame, op_ret, op_errno, buf, stbuf,
+                            xdata);
         return 0;
 }
 
-
 int
 trace_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno,
                   inode_t *inode, struct iatt *buf,
                   dict_t *xdata, struct iatt *postparent)
 {
-        char  *statstr = NULL;
-        char  *postparentstr = NULL;
+        char          statstr[4096]       = {0, };
+        char          postparentstr[4096] = {0, };
+        trace_conf_t  *conf               = NULL;
 
-        if (trace_fop_names[GF_FOP_LOOKUP].enabled) {
-                if (op_ret >= 0) {
-                        statstr = trace_stat_to_str (buf);
-                        postparentstr = trace_stat_to_str (postparent);
+        conf = this->private;
 
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s (op_ret=%d "
-                                "*buf {%s}, *postparent {%s}",
-                                frame->root->unique, uuid_utoa (inode->gfid),
-                                op_ret, statstr, postparentstr);
-
-                        if (statstr)
-                                GF_FREE (statstr);
-                        if (postparentstr)
-                                GF_FREE (postparentstr);
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_LOOKUP].enabled) {
+                char  string[4096] = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (buf, statstr);
+                        trace_stat_to_str (postparent, postparentstr);
+                        /* print buf->ia_gfid instead of inode->gfid,
+                         * since if the inode is not yet linked to the
+                         * inode table (fresh lookup) then null gfid
+                         * will be printed.
+                         */
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s (op_ret=%d "
+                                  "*buf {%s}, *postparent {%s}",
+                                  frame->root->unique,
+                                  uuid_utoa (buf->ia_gfid),
+                                  op_ret, statstr, postparentstr);
 
                         /* For 'forget' */
                         inode_ctx_put (inode, this, 0);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d, op_errno=%d)",
-                                frame->root->unique, uuid_utoa (frame->local),
-                                op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d)",
+                                  frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode, buf,
-                             xdata, postparent);
+out:
+        TRACE_STACK_UNWIND (lookup, frame, op_ret, op_errno, inode, buf,
+                            xdata, postparent);
         return 0;
 }
 
-
 int
 trace_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno,
                    inode_t *inode, struct iatt *buf,
-                   struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+                   struct iatt *preparent, struct iatt *postparent,
+                   dict_t *xdata)
 {
-        char  *statstr = NULL;
-        char  *preparentstr = NULL;
-        char  *postparentstr = NULL;
-
-        if (trace_fop_names[GF_FOP_SYMLINK].enabled) {
-                if (op_ret >= 0) {
-                        statstr = trace_stat_to_str (buf);
-                        preparentstr = trace_stat_to_str (preparent);
-                        postparentstr = trace_stat_to_str (postparent);
+        char          statstr[4096]       = {0, };
+        char          preparentstr[4096]  = {0, };
+        char          postparentstr[4096] = {0, };
+        trace_conf_t  *conf               = NULL;
 
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s (op_ret=%d "
-                                "*stbuf = {%s}, *preparent = {%s}, "
-                                "*postparent = {%s})",
-                                frame->root->unique, uuid_utoa (inode->gfid),
-                                op_ret, statstr, preparentstr, postparentstr);
-
-                        if (statstr)
-                                GF_FREE (statstr);
-
-                        if (preparentstr)
-                                GF_FREE (preparentstr);
-
-                        if (postparentstr)
-                                GF_FREE (postparentstr);
+        conf = this->private;
 
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_SYMLINK].enabled) {
+                char  string[4096] = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (buf, statstr);
+                        trace_stat_to_str (preparent, preparentstr);
+                        trace_stat_to_str (postparent, postparentstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s (op_ret=%d "
+                                  "*stbuf = {%s}, *preparent = {%s}, "
+                                  "*postparent = {%s})",
+                                  frame->root->unique,
+                                  uuid_utoa (inode->gfid),
+                                  op_ret, statstr, preparentstr,
+                                  postparentstr);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": op_ret=%d, op_errno=%d",
-                                frame->root->unique, op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": op_ret=%d, op_errno=%d",
+                                  frame->root->unique, op_ret,
+                                  op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (symlink, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent, xdata);
+out:
+        TRACE_STACK_UNWIND (symlink, frame, op_ret, op_errno, inode, buf,
+                            preparent, postparent, xdata);
         return 0;
 }
 
-
 int
 trace_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno,
                  inode_t *inode, struct iatt *buf,
                  struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-        char *statstr = NULL;
-        char *preparentstr = NULL;
-        char *postparentstr = NULL;
-
-        if (trace_fop_names[GF_FOP_MKNOD].enabled) {
-                if (op_ret >= 0) {
-                        statstr = trace_stat_to_str (buf);
-                        preparentstr = trace_stat_to_str (preparent);
-                        postparentstr = trace_stat_to_str (postparent);
-
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s (op_ret=%d "
-                                "*stbuf = {%s}, *preparent = {%s}, "
-                                "*postparent = {%s})",
-                                frame->root->unique, uuid_utoa (inode->gfid),
-                                op_ret, statstr, preparentstr, postparentstr);
-
-                        if (statstr)
-                                GF_FREE (statstr);
+        char          statstr[4096]       = {0, };
+        char          preparentstr[4096]  = {0, };
+        char          postparentstr[4096] = {0, };
+        trace_conf_t  *conf               = NULL;
 
-                        if (preparentstr)
-                                GF_FREE (preparentstr);
+        conf = this->private;
 
-                        if (postparentstr)
-                                GF_FREE (postparentstr);
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        char string[4096]  = {0,};
+        if (trace_fop_names[GF_FOP_MKNOD].enabled) {
+                if (op_ret == 0) {
+                        trace_stat_to_str (buf, statstr);
+                        trace_stat_to_str (preparent, preparentstr);
+                        trace_stat_to_str (postparent, postparentstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s (op_ret=%d "
+                                  "*stbuf = {%s}, *preparent = {%s}, "
+                                  "*postparent = {%s})",
+                                  frame->root->unique,
+                                  uuid_utoa (inode->gfid),
+                                  op_ret, statstr, preparentstr,
+                                  postparentstr);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": (op_ret=%d, op_errno=%d)",
-                                frame->root->unique, op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, op_errno=%d)",
+                                  frame->root->unique, op_ret,
+                                  op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (mknod, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent, xdata);
+out:
+        TRACE_STACK_UNWIND (mknod, frame, op_ret, op_errno, inode, buf,
+                            preparent, postparent, xdata);
         return 0;
 }
 
-
 int
 trace_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno,
                  inode_t *inode, struct iatt *buf,
                  struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-        char  *statstr = NULL;
-        char  *preparentstr = NULL;
-        char  *postparentstr = NULL;
-
-        if (trace_fop_names[GF_FOP_MKDIR].enabled) {
-                if (op_ret >= 0) {
-                        statstr = trace_stat_to_str (buf);
-                        preparentstr = trace_stat_to_str (preparent);
-                        postparentstr = trace_stat_to_str (postparent);
-
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s (op_ret=%d "
-                                ", *stbuf = {%s}, *prebuf = {%s}, "
-                                "*postbuf = {%s} )",
-                                frame->root->unique, uuid_utoa (inode->gfid),
-                                op_ret, statstr, preparentstr, postparentstr);
-
-                        if (statstr)
-                                GF_FREE (statstr);
+        char          statstr[4096]       = {0, };
+        char          preparentstr[4096]  = {0, };
+        char          postparentstr[4096] = {0, };
+        trace_conf_t  *conf               = NULL;
 
-                        if (preparentstr)
-                                GF_FREE (preparentstr);
+        conf = this->private;
 
-                        if (postparentstr)
-                                GF_FREE (postparentstr);
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_MKDIR].enabled) {
+                char  string[4096]  = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (buf, statstr);
+                        trace_stat_to_str (preparent, preparentstr);
+                        trace_stat_to_str (postparent, postparentstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s (op_ret=%d "
+                                  ", *stbuf = {%s}, *prebuf = {%s}, "
+                                  "*postbuf = {%s} )",
+                                  frame->root->unique,
+                                  uuid_utoa (inode->gfid),
+                                  op_ret, statstr, preparentstr,
+                                  postparentstr);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": (op_ret=%d, op_errno=%d)",
-                                frame->root->unique, op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, op_errno=%d)",
+                                  frame->root->unique, op_ret,
+                                  op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (mkdir, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent, xdata);
+out:
+        TRACE_STACK_UNWIND (mkdir, frame, op_ret, op_errno, inode, buf,
+                            preparent, postparent, xdata);
         return 0;
 }
 
-
 int
 trace_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 int32_t op_ret, int32_t op_errno,
                 inode_t *inode, struct iatt *buf,
                 struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-        char  *statstr = NULL;
-        char  *preparentstr = NULL;
-        char  *postparentstr = NULL;
+        char          statstr[4096]       = {0, };
+        char          preparentstr[4096]  = {0, };
+        char          postparentstr[4096] = {0, };
+        trace_conf_t  *conf               = NULL;
 
-        if (trace_fop_names[GF_FOP_LINK].enabled) {
-                if (op_ret >= 0) {
-                        statstr = trace_stat_to_str (buf);
-                        preparentstr = trace_stat_to_str (preparent);
-                        postparentstr = trace_stat_to_str (postparent);
+        conf = this->private;
 
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": (op_ret=%d, *stbuf = {%s}, "
-                                " *prebuf = {%s}, *postbuf = {%s})",
-                                frame->root->unique, op_ret,
-                                statstr, preparentstr, postparentstr);
-
-                        if (statstr)
-                                GF_FREE (statstr);
-
-                        if (preparentstr)
-                                GF_FREE (preparentstr);
-
-                        if (postparentstr)
-                                GF_FREE (postparentstr);
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        char  string[4096]  = {0,};
+        if (trace_fop_names[GF_FOP_LINK].enabled) {
+                if (op_ret == 0) {
+                        trace_stat_to_str (buf, statstr);
+                        trace_stat_to_str (preparent, preparentstr);
+                        trace_stat_to_str (postparent, postparentstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, "
+                                  "*stbuf = {%s},  *prebuf = {%s},"
+                                  " *postbuf = {%s})",
+                                  frame->root->unique, op_ret,
+                                  statstr, preparentstr, postparentstr);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                                frame->root->unique, uuid_utoa (frame->local),
-                                op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d",
+                                  frame->root->unique,
+                                  uuid_utoa (frame->local),
+                                  op_ret, op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (link, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent, xdata);
+out:
+        TRACE_STACK_UNWIND (link, frame, op_ret, op_errno, inode, buf,
+                            preparent, postparent, xdata);
         return 0;
 }
 
-
 int
 trace_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        char      string[4096] = {0,};
         if (trace_fop_names[GF_FOP_FLUSH].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                        frame->root->unique, uuid_utoa (frame->local),
-                        op_ret, op_errno);
-        }
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique, uuid_utoa (frame->local),
+                          op_ret, op_errno);
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno, xdata);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (flush, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
-
 int
 trace_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        char    string[4096] = {0,};
         if (trace_fop_names[GF_FOP_OPENDIR].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s op_ret=%d, op_errno=%d, fd=%p",
-                        frame->root->unique, uuid_utoa (frame->local),
-                        op_ret, op_errno, fd);
-        }
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d,"
+                          " fd=%p",
+                          frame->root->unique, uuid_utoa (frame->local),
+                          op_ret, op_errno, fd);
 
+                LOG_ELEMENT (conf, string);
+        }
+out:
         /* for 'releasedir' log */
         if (op_ret >= 0)
                 fd_ctx_set (fd, this, 0);
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (opendir, frame, op_ret, op_errno, fd, xdata);
+        TRACE_STACK_UNWIND (opendir, frame, op_ret, op_errno, fd, xdata);
         return 0;
 }
 
-
 int
 trace_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno,
                  struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-        char  *preparentstr = NULL;
-        char  *postparentstr = NULL;
-
-        if (trace_fop_names[GF_FOP_RMDIR].enabled) {
-                if (op_ret >= 0) {
-                        preparentstr = trace_stat_to_str (preparent);
-                        postparentstr = trace_stat_to_str (postparent);
-
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d, *prebuf = {%s}, "
-                                "*postbuf = {%s}",
-                                frame->root->unique, uuid_utoa (frame->local),
-                                op_ret, preparentstr,  postparentstr);
+        char           preparentstr[4096]  = {0, };
+        char           postparentstr[4096] = {0, };
+        trace_conf_t   *conf               = NULL;
 
-                        if (preparentstr)
-                                GF_FREE (preparentstr);
+        conf = this->private;
 
-                        if (postparentstr)
-                                GF_FREE (postparentstr);
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_RMDIR].enabled) {
+                char  string[4096] = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (preparent, preparentstr);
+                        trace_stat_to_str (postparent, postparentstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "*prebuf={%s},  *postbuf={%s}",
+                                  frame->root->unique,
+                                  uuid_utoa (frame->local),
+                                  op_ret, preparentstr, postparentstr);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                                frame->root->unique, uuid_utoa (frame->local),
-                                op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d", frame->root->unique,
+                                  uuid_utoa (frame->local),
+                                  op_ret, op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (rmdir, frame, op_ret, op_errno,
-                             preparent, postparent, xdata);
+out:
+        TRACE_STACK_UNWIND (rmdir, frame, op_ret, op_errno,
+                            preparent, postparent, xdata);
         return 0;
 }
 
-
 int
 trace_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno,
                     struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
 {
-        char  *preopstr = NULL;
-        char  *postopstr = NULL;
+        char           preopstr[4096]  = {0, };
+        char           postopstr[4096] = {0, };
+        trace_conf_t   *conf           = NULL;
 
-        if (trace_fop_names[GF_FOP_TRUNCATE].enabled) {
-                if (op_ret >= 0) {
-                        preopstr = trace_stat_to_str (prebuf);
-                        postopstr = trace_stat_to_str (postbuf);
-
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": (op_ret=%d, *prebuf = {%s}, "
-                                "*postbuf = {%s} )",
-                                frame->root->unique, op_ret, preopstr,
-                                postopstr);
+        conf = this->private;
 
-                        if (preopstr)
-                                GF_FREE (preopstr);
-
-                        if (postopstr)
-                                GF_FREE (postopstr);
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_TRUNCATE].enabled) {
+                char   string[4096] = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (prebuf, preopstr);
+                        trace_stat_to_str (postbuf, postopstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, "
+                                  "*prebuf = {%s}, *postbuf = {%s} )",
+                                  frame->root->unique, op_ret,
+                                  preopstr, postopstr);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                                frame->root->unique, uuid_utoa (frame->local),
-                                op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d", frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno, prebuf, postbuf, xdata);
+out:
+        TRACE_STACK_UNWIND (truncate, frame, op_ret, op_errno, prebuf,
+                            postbuf, xdata);
         return 0;
 }
 
-
 int
 trace_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct statvfs *buf, dict_t *xdata)
+                  int32_t op_ret, int32_t op_errno, struct statvfs *buf,
+                  dict_t *xdata)
 {
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_STATFS].enabled) {
-                if (op_ret >= 0) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": ({f_bsize=%lu, f_frsize=%lu, f_blocks=%"GF_PRI_FSBLK
-                                ", f_bfree=%"GF_PRI_FSBLK", f_bavail=%"GF_PRI_FSBLK", "
-                                "f_files=%"GF_PRI_FSBLK", f_ffree=%"GF_PRI_FSBLK", f_favail=%"
-                                GF_PRI_FSBLK", f_fsid=%lu, f_flag=%lu, f_namemax=%lu}) => ret=%d",
-                                frame->root->unique, buf->f_bsize, buf->f_frsize, buf->f_blocks,
-                                buf->f_bfree, buf->f_bavail, buf->f_files, buf->f_ffree,
-                                buf->f_favail, buf->f_fsid, buf->f_flag, buf->f_namemax, op_ret);
+                char   string[4096] = {0,};
+                if (op_ret == 0) {
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": ({f_bsize=%lu, "
+                                  "f_frsize=%lu, "
+                                  "f_blocks=%"GF_PRI_FSBLK
+                                  ", f_bfree=%"GF_PRI_FSBLK", "
+                                  "f_bavail=%"GF_PRI_FSBLK", "
+                                  "f_files=%"GF_PRI_FSBLK", "
+                                  "f_ffree=%"GF_PRI_FSBLK", "
+                                  "f_favail=%"GF_PRI_FSBLK", "
+                                  "f_fsid=%lu, f_flag=%lu, "
+                                  "f_namemax=%lu}) => ret=%d",
+                                  frame->root->unique, buf->f_bsize,
+                                  buf->f_frsize, buf->f_blocks,
+                                  buf->f_bfree, buf->f_bavail,
+                                  buf->f_files, buf->f_ffree,
+                                  buf->f_favail, buf->f_fsid,
+                                  buf->f_flag, buf->f_namemax, op_ret);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": (op_ret=%d, op_errno=%d)",
-                                frame->root->unique, op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, "
+                                  "op_errno=%d)",
+                                  frame->root->unique, op_ret,
+                                  op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (statfs, frame, op_ret, op_errno, buf, xdata);
+out:
+        TRACE_STACK_UNWIND (statfs, frame, op_ret, op_errno, buf, xdata);
         return 0;
 }
 
-
 int
 trace_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
+        trace_conf_t     *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_SETXATTR].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                        frame->root->unique, uuid_utoa (frame->local),
-                        op_ret, op_errno);
-        }
+                char      string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique,
+                          uuid_utoa (frame->local), op_ret,
+                          op_errno);
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, xdata);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (setxattr, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
-
 int
 trace_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+                    int32_t op_ret, int32_t op_errno, dict_t *dict,
+                    dict_t *xdata)
 {
+        trace_conf_t    *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_GETXATTR].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s op_ret=%d, op_errno=%d, dict=%p",
-                        frame->root->unique, uuid_utoa (frame->local), op_ret,
-                        op_errno, dict);
-        }
+                char       string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d,"
+                          " dict=%p", frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno,
+                          dict);
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict, xdata);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, xdata);
 
         return 0;
 }
@@ -928,32 +997,49 @@ int
 trace_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+                goto out;
         if (trace_fop_names[GF_FOP_FSETXATTR].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                        frame->root->unique, uuid_utoa (frame->local), op_ret,
-                        op_errno);
-        }
+                char    string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno);
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, xdata);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
-
 int
 trace_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+                     int32_t op_ret, int32_t op_errno, dict_t *dict,
+                     dict_t *xdata)
 {
+        trace_conf_t    *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FGETXATTR].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s op_ret=%d, op_errno=%d, dict=%p",
-                        frame->root->unique, uuid_utoa (frame->local), op_ret,
-                        op_errno, dict);
-        }
+                char      string[4096]  = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d,"
+                          " dict=%p", frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno,
+                          dict);
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (fgetxattr, frame, op_ret, op_errno, dict, xdata);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, dict, xdata);
 
         return 0;
 }
@@ -962,163 +1048,208 @@ int
 trace_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_REMOVEXATTR].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                        frame->root->unique, uuid_utoa (frame->local),
-                        op_ret, op_errno);
-        }
+                char       string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno);
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno, xdata);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (removexattr, frame, op_ret, op_errno, xdata);
 
         return 0;
 }
 
-
 int
 trace_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
+        trace_conf_t     *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FSYNCDIR].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                        frame->root->unique, uuid_utoa (frame->local),
-                        op_ret, op_errno);
-        }
+                char       string[4096]   =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno);
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (fsyncdir, frame, op_ret, op_errno, xdata);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (fsyncdir, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
-
 int
 trace_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_ACCESS].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s op_ret=%d, op_errno=%d)",
-                        frame->root->unique, uuid_utoa (frame->local),
-                        op_ret, op_errno);
-        }
+                char      string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, "
+                          "op_errno=%d)", frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno);
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (access, frame, op_ret, op_errno, xdata);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (access, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
-
 int
 trace_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno,
                      struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
 {
-        char  *prebufstr = NULL;
-        char  *postbufstr = NULL;
-
-        if (trace_fop_names[GF_FOP_FTRUNCATE].enabled) {
-                if (op_ret >= 0) {
-                        prebufstr = trace_stat_to_str (prebuf);
-                        postbufstr = trace_stat_to_str (postbuf);
-
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": op_ret=%d, *prebuf = {%s}, "
-                                "*postbuf = {%s} )",
-                                frame->root->unique, op_ret,
-                                prebufstr, postbufstr);
+        char          prebufstr[4096]  = {0, };
+        char          postbufstr[4096] = {0, };
+        trace_conf_t  *conf            = NULL;
 
-                        if (prebufstr)
-                                GF_FREE (prebufstr);
-
-                        if (postbufstr)
-                                GF_FREE (postbufstr);
+        conf = this->private;
 
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_FTRUNCATE].enabled) {
+                char  string[4096]  = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (prebuf, prebufstr);
+                        trace_stat_to_str (postbuf, postbufstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": op_ret=%d, "
+                                  "*prebuf = {%s}, *postbuf = {%s} )",
+                                  frame->root->unique, op_ret,
+                                  prebufstr, postbufstr);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                                frame->root->unique, uuid_utoa (frame->local),
-                                op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d", frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (ftruncate, frame, op_ret, op_errno, prebuf, postbuf, xdata);
+out:
+        TRACE_STACK_UNWIND (ftruncate, frame, op_ret, op_errno, prebuf, postbuf,
+                            xdata);
         return 0;
 }
 
-
 int
 trace_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
-        char *statstr = NULL;
+        char          statstr[4096] = {0, };
+        trace_conf_t  *conf         = NULL;
+
+        conf = this->private;
 
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FSTAT].enabled) {
-                if (op_ret >= 0) {
-                        statstr = trace_stat_to_str (buf);
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d buf=%s",
-                                frame->root->unique, uuid_utoa (frame->local),
-                                op_ret, statstr);
-
-                        if (statstr)
-                                GF_FREE (statstr);
+                char string[4096]  = {0.};
+                if (op_ret == 0) {
+                        trace_stat_to_str (buf, statstr);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d "
+                                  "buf=%s", frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  statstr);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                                frame->root->unique, uuid_utoa (frame->local),
-                                op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d", frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (fstat, frame, op_ret, op_errno, buf, xdata);
+out:
+        TRACE_STACK_UNWIND (fstat, frame, op_ret, op_errno, buf, xdata);
         return 0;
 }
 
-
 int
 trace_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int32_t op_ret, int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
+              int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+              dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_LK].enabled) {
-                if (op_ret >= 0) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d, {l_type=%d, l_whence=%d, "
-                                "l_start=%"PRId64", l_len=%"PRId64", l_pid=%u})",
-                                frame->root->unique, uuid_utoa (frame->local),
-                                op_ret, lock->l_type, lock->l_whence,
-                                lock->l_start, lock->l_len, lock->l_pid);
+                char      string[4096] = {0,};
+                if (op_ret == 0) {
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "{l_type=%d, l_whence=%d, "
+                                  "l_start=%"PRId64", "
+                                  "l_len=%"PRId64", l_pid=%u})",
+                                  frame->root->unique,
+                                  uuid_utoa (frame->local),
+                                  op_ret, lock->l_type, lock->l_whence,
+                                  lock->l_start, lock->l_len,
+                                  lock->l_pid);
                 } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s op_ret=%d, op_errno=%d)",
-                                frame->root->unique, uuid_utoa (frame->local),
-                                op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d)", frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
                 }
-        }
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (lk, frame, op_ret, op_errno, lock, xdata);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (lk, frame, op_ret, op_errno, lock, xdata);
         return 0;
 }
 
-
-
 int
 trace_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_ENTRYLK].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                        frame->root->unique, uuid_utoa (frame->local),
-                        op_ret, op_errno);
-        }
+                char   string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno);
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (entrylk, frame, op_ret, op_errno, xdata);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (entrylk, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
@@ -1126,66 +1257,97 @@ int
 trace_fentrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
+        trace_conf_t    *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FENTRYLK].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                        frame->root->unique, uuid_utoa (frame->local),
-                        op_ret, op_errno);
-        }
+                char      string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno);
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (fentrylk, frame, op_ret, op_errno, xdata);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (fentrylk, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
-
 int
 trace_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+                   int32_t op_ret, int32_t op_errno, dict_t *dict,
+                   dict_t *xdata)
 {
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_XATTROP].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                        frame->root->unique, uuid_utoa (frame->local),
-                        op_ret, op_errno);
-        }
+                char    string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno);
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (xattrop, frame, op_ret, op_errno, dict, xdata);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (xattrop, frame, op_ret, op_errno, dict, xdata);
         return 0;
 }
 
-
 int
 trace_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+                    int32_t op_ret, int32_t op_errno, dict_t *dict,
+                    dict_t *xdata)
 {
+        trace_conf_t    *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FXATTROP].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                        frame->root->unique, uuid_utoa (frame->local),
-                        op_ret, op_errno);
-        }
+                char      string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno);
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (fxattrop, frame, op_ret, op_errno, dict, xdata);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (fxattrop, frame, op_ret, op_errno, dict, xdata);
         return 0;
 }
 
-
 int
 trace_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_INODELK].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                        frame->root->unique, uuid_utoa (frame->local),
-                        op_ret, op_errno);
-        }
+                char       string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique,
+                          uuid_utoa (frame->local),op_ret, op_errno);
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (inodelk, frame, op_ret, op_errno, xdata);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (inodelk, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
@@ -1193,32 +1355,51 @@ int
 trace_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FINODELK].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                        frame->root->unique, uuid_utoa (frame->local), op_ret, op_errno);
-        }
+                char      string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno);
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (finodelk, frame, op_ret, op_errno, xdata);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (finodelk, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
-
 int
 trace_rchecksum_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno,
-                     uint32_t weak_checksum, uint8_t *strong_checksum, dict_t *xdata)
+                     uint32_t weak_checksum, uint8_t *strong_checksum,
+                     dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+                goto out;
         if (trace_fop_names[GF_FOP_RCHECKSUM].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s op_ret=%d op_errno=%d",
-                        frame->root->unique, uuid_utoa (frame->local), op_ret, op_errno);
+                char      string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d op_errno=%d",
+                          frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno);
+
+                LOG_ELEMENT (conf, string);
         }
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (rchecksum, frame, op_ret, op_errno, weak_checksum,
-                             strong_checksum, xdata);
+out:
+        TRACE_STACK_UNWIND (rchecksum, frame, op_ret, op_errno, weak_checksum,
+                            strong_checksum, xdata);
 
         return 0;
 }
@@ -1230,17 +1411,31 @@ trace_entrylk (call_frame_t *frame, xlator_t *this,
                const char *volume, loc_t *loc, const char *basename,
                entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_ENTRYLK].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s volume=%s, (path=%s basename=%s, "
-                        "cmd=%s, type=%s)",
-                        frame->root->unique, uuid_utoa (loc->inode->gfid),
-                        volume, loc->path, basename,
-                        ((cmd == ENTRYLK_LOCK) ? "ENTRYLK_LOCK" : "ENTRYLK_UNLOCK"),
-                        ((type == ENTRYLK_RDLCK) ? "ENTRYLK_RDLCK" : "ENTRYLK_WRLCK"));
+                char     string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s volume=%s, (path=%s "
+                          "basename=%s, cmd=%s, type=%s)",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid),
+                          volume, loc->path, basename,
+                          ((cmd == ENTRYLK_LOCK) ? "ENTRYLK_LOCK" :
+                           "ENTRYLK_UNLOCK"),
+                          ((type == ENTRYLK_RDLCK) ? "ENTRYLK_RDLCK" :
+                           "ENTRYLK_WRLCK"));
+
                 frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_entrylk_cbk,
                     FIRST_CHILD (this),
                     FIRST_CHILD (this)->fops->entrylk,
@@ -1248,15 +1443,20 @@ trace_entrylk (call_frame_t *frame, xlator_t *this,
         return 0;
 }
 
-
 int
 trace_inodelk (call_frame_t *frame, xlator_t *this, const char *volume,
                loc_t *loc, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
 {
-        char *cmd_str = NULL;
-        char *type_str = NULL;
+        char         *cmd_str  = NULL;
+        char         *type_str = NULL;
+        trace_conf_t *conf     = NULL;
 
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_INODELK].enabled) {
+                char string[4096]  = {0,};
                 switch (cmd) {
 #if F_GETLK != F_GETLK64
                 case F_GETLK64:
@@ -1299,17 +1499,22 @@ trace_inodelk (call_frame_t *frame, xlator_t *this, const char *volume,
                         break;
                 }
 
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s volume=%s, (path=%s "
-                        "cmd=%s, type=%s, start=%llu, len=%llu, pid=%llu)",
-                        frame->root->unique, uuid_utoa (loc->inode->gfid),
-                        volume, loc->path,
-                        cmd_str, type_str, (unsigned long long) flock->l_start,
-                        (unsigned long long) flock->l_len,
-                        (unsigned long long) flock->l_pid);
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s volume=%s, (path=%s "
+                          "cmd=%s, type=%s, start=%llu, len=%llu, "
+                          "pid=%llu)", frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), volume,
+                          loc->path, cmd_str, type_str,
+                          (unsigned long long)flock->l_start,
+                          (unsigned long long) flock->l_len,
+                          (unsigned long long) flock->l_pid);
+
                 frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_inodelk_cbk,
                     FIRST_CHILD (this),
                     FIRST_CHILD (this)->fops->inodelk,
@@ -1317,14 +1522,20 @@ trace_inodelk (call_frame_t *frame, xlator_t *this, const char *volume,
         return 0;
 }
 
-
 int
 trace_finodelk (call_frame_t *frame, xlator_t *this, const char *volume,
                 fd_t *fd, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
 {
-        char *cmd_str = NULL, *type_str = NULL;
+        char          *cmd_str  = NULL;
+        char          *type_str = NULL;
+        trace_conf_t  *conf     = NULL;
 
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FINODELK].enabled) {
+                char  string[4096] = {0,};
                 switch (cmd) {
 #if F_GETLK != F_GETLK64
                 case F_GETLK64:
@@ -1367,16 +1578,21 @@ trace_finodelk (call_frame_t *frame, xlator_t *this, const char *volume,
                         break;
                 }
 
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s volume=%s, (fd =%p "
-                        "cmd=%s, type=%s, start=%llu, len=%llu, pid=%llu)",
-                        frame->root->unique, uuid_utoa (fd->inode->gfid), volume, fd,
-                        cmd_str, type_str, (unsigned long long) flock->l_start,
-                        (unsigned long long) flock->l_len,
-                        (unsigned long long) flock->l_pid);
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s volume=%s, (fd =%p "
+                          "cmd=%s, type=%s, start=%llu, len=%llu, "
+                          "pid=%llu)", frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), volume, fd,
+                          cmd_str, type_str,
+                          (unsigned long long) flock->l_start,
+                          (unsigned long long) flock->l_len,
+                          (unsigned long long) flock->l_pid);
+
                 frame->local = fd->inode->gfid;
-        }
 
+                LOG_ELEMENT (conf, string);
+        }
+out:
         STACK_WIND (frame, trace_finodelk_cbk,
                     FIRST_CHILD (this),
                     FIRST_CHILD (this)->fops->finodelk,
@@ -1384,19 +1600,30 @@ trace_finodelk (call_frame_t *frame, xlator_t *this, const char *volume,
         return 0;
 }
 
-
 int
 trace_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
                gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_XATTROP].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s (path=%s flags=%d)",
-                        frame->root->unique, uuid_utoa (loc->inode->gfid),
-                        loc->path, flags);
+                char    string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s (path=%s flags=%d)",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          flags);
+
                 frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_xattrop_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->xattrop,
@@ -1405,19 +1632,29 @@ trace_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
         return 0;
 }
 
-
 int
 trace_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
                 gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
+        trace_conf_t    *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FXATTROP].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s fd=%p, flags=%d",
-                        frame->root->unique, uuid_utoa (fd->inode->gfid),
-                        fd, flags);
+                char    string[4096]  = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s fd=%p, flags=%d",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), fd, flags);
+
                 frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_fxattrop_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fxattrop,
@@ -1426,20 +1663,30 @@ trace_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
         return 0;
 }
 
-
 int
 trace_lookup (call_frame_t *frame, xlator_t *this,
               loc_t *loc, dict_t *xdata)
 {
+        trace_conf_t *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_LOOKUP].enabled) {
+                char string[4096] = {0,};
                 /* TODO: print all the keys mentioned in xattr_req */
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s path=%s",
-                        frame->root->unique, uuid_utoa (loc->inode->gfid),
-                        loc->path);
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path);
+
                 frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_lookup_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->lookup,
@@ -1448,18 +1695,28 @@ trace_lookup (call_frame_t *frame, xlator_t *this,
         return 0;
 }
 
-
 int
 trace_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+                goto out;
         if (trace_fop_names[GF_FOP_STAT].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s path=%s",
-                        frame->root->unique, uuid_utoa (loc->inode->gfid),
-                        loc->path);
+                char  string[4096]  = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path);
+
                 frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_stat_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->stat,
@@ -1468,18 +1725,30 @@ trace_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
         return 0;
 }
 
-
 int
-trace_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size, dict_t *xdata)
+trace_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+                dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_READLINK].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s path=%s, size=%"GF_PRI_SIZET")",
-                        frame->root->unique, uuid_utoa (loc->inode->gfid),
-                        loc->path, size);
+                char      string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s, "
+                          "size=%"GF_PRI_SIZET")", frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          size);
+
                 frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_readlink_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->readlink,
@@ -1488,19 +1757,29 @@ trace_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size, di
         return 0;
 }
 
-
 int
 trace_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc,
              mode_t mode, dev_t dev, mode_t umask, dict_t *xdata)
 {
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+                goto out;
         if (trace_fop_names[GF_FOP_MKNOD].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s path=%s mode=0%o umask=0%o "
-                        "dev=%"GF_PRI_DEV")",
-                        frame->root->unique, uuid_utoa (loc->inode->gfid),
-                        loc->path, mode, umask, dev);
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s mode=%d "
+                          "umask=0%o, dev=%"GF_PRI_DEV")",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          mode, umask, dev);
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_mknod_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->mknod,
@@ -1509,18 +1788,28 @@ trace_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc,
         return 0;
 }
 
-
 int
 trace_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
              mode_t umask, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_MKDIR].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s path=%s mode=0%o umask=0%o",
-                        frame->root->unique, uuid_utoa (loc->inode->gfid),
-                        loc->path, mode, umask);
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s mode=%d"
+                          " umask=0%o", frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          mode, umask);
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_mkdir_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->mkdir,
@@ -1528,19 +1817,29 @@ trace_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
         return 0;
 }
 
-
 int
 trace_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
               dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_UNLINK].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s path=%s flag=%d",
-                        frame->root->unique, uuid_utoa (loc->inode->gfid),
-                        loc->path, xflag);
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s flag=%d",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          xflag);
+
                 frame->local = loc->inode->gfid;
-        }
 
+                LOG_ELEMENT (conf, string);
+        }
+out:
         STACK_WIND (frame, trace_unlink_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->unlink,
@@ -1548,19 +1847,30 @@ trace_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
         return 0;
 }
 
-
 int
 trace_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
              dict_t *xdata)
 {
+        trace_conf_t    *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_RMDIR].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s path=%s flags=%d",
-                        frame->root->unique, uuid_utoa (loc->inode->gfid),
-                        loc->path, flags);
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s flags=%d",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          flags);
+
                 frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_rmdir_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->rmdir,
@@ -1569,18 +1879,28 @@ trace_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
         return 0;
 }
 
-
 int
 trace_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
                loc_t *loc, mode_t umask, dict_t *xdata)
 {
+        trace_conf_t     *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_SYMLINK].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s linkpath=%s, path=%s umask=0%o",
-                        frame->root->unique, uuid_utoa (loc->inode->gfid),
-                        linkpath, loc->path, umask);
+                char     string[4096]   =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s linkpath=%s, path=%s"
+                          " umask=0%o", frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), linkpath,
+                          loc->path, umask);
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_symlink_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->symlink,
@@ -1589,15 +1909,20 @@ trace_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
         return 0;
 }
 
-
 int
 trace_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
               dict_t *xdata)
 {
-        char oldgfid[50] = {0,};
-        char newgfid[50] = {0,};
+        char         oldgfid[50] = {0,};
+        char         newgfid[50] = {0,};
+        trace_conf_t *conf       = NULL;
 
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_RENAME].enabled) {
+                char string[4096] = {0,};
                 if (newloc->inode)
                         uuid_utoa_r (newloc->inode->gfid, newgfid);
                 else
@@ -1605,13 +1930,18 @@ trace_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
 
                 uuid_utoa_r (oldloc->inode->gfid, oldgfid);
 
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": oldgfid=%s oldpath=%s --> newgfid=%s newpath=%s",
-                        frame->root->unique, oldgfid, oldloc->path, newgfid, newloc->path);
+                snprintf (string, sizeof (string),
+                          "%"PRId64": oldgfid=%s oldpath=%s --> "
+                          "newgfid=%s newpath=%s",
+                          frame->root->unique, oldgfid,
+                          oldloc->path, newgfid, newloc->path);
 
                 frame->local = oldloc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_rename_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->rename,
@@ -1620,15 +1950,20 @@ trace_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
         return 0;
 }
 
-
 int
 trace_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
             dict_t *xdata)
 {
-        char oldgfid[50] = {0,};
-        char newgfid[50] = {0,};
+        char         oldgfid[50] = {0,};
+        char         newgfid[50] = {0,};
+        trace_conf_t *conf       = NULL;
+
+        conf = this->private;
 
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_LINK].enabled) {
+                char string[4096]  = {0,};
                 if (newloc->inode)
                         uuid_utoa_r (newloc->inode->gfid, newgfid);
                 else
@@ -1636,13 +1971,18 @@ trace_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
 
                 uuid_utoa_r (oldloc->inode->gfid, oldgfid);
 
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": oldgfid=%s oldpath=%s --> newgfid=%s newpath=%s",
-                        frame->root->unique, oldgfid, oldloc->path,
-                        newgfid, newloc->path);
+                snprintf (string, sizeof (string),
+                          "%"PRId64": oldgfid=%s oldpath=%s --> "
+                          "newgfid=%s newpath=%s", frame->root->unique,
+                          oldgfid, oldloc->path, newgfid,
+                          newloc->path);
+
                 frame->local = oldloc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_link_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->link,
@@ -1650,43 +1990,69 @@ trace_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
         return 0;
 }
 
-
 int
 trace_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
                struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        char      actime_str[64]   = {0,};
-        char      modtime_str[64]  = {0,};
+        uint64_t     ia_time          = 0;
+        char         actime_str[256]  = {0,};
+        char         modtime_str[256] = {0,};
+        trace_conf_t *conf            = NULL;
 
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_SETATTR].enabled) {
+                char     string[4096]  =  {0,};
                 if (valid & GF_SET_ATTR_MODE) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s path=%s mode=%o)",
-                                frame->root->unique, uuid_utoa (loc->inode->gfid),
-                                loc->path, st_mode_from_ia (stbuf->ia_prot, stbuf->ia_type));
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s path=%s mode=%o)",
+                                  frame->root->unique,
+                                  uuid_utoa (loc->inode->gfid),
+                                  loc->path,
+                                  st_mode_from_ia (stbuf->ia_prot,
+                                                   stbuf->ia_type));
+
+                        LOG_ELEMENT (conf, string);
+                        memset (string, 0 , sizeof (string));
                 }
 
                 if (valid & (GF_SET_ATTR_UID | GF_SET_ATTR_GID)) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s path=%s uid=%o, gid=%o",
-                                frame->root->unique,  uuid_utoa (loc->inode->gfid),
-                                loc->path, stbuf->ia_uid, stbuf->ia_gid);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s path=%s uid=%o,"
+                                  " gid=%o", frame->root->unique,
+                                  uuid_utoa (loc->inode->gfid),
+                                  loc->path, stbuf->ia_uid,
+                                  stbuf->ia_gid);
+
+                        LOG_ELEMENT (conf, string);
+                        memset (string, 0 , sizeof (string));
                 }
 
                 if (valid & (GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME)) {
-                        gf_time_fmt (actime_str, sizeof actime_str,
-                                     stbuf->ia_atime, gf_timefmt_bdT);
-                        gf_time_fmt (modtime_str, sizeof modtime_str,
-                                     stbuf->ia_mtime, gf_timefmt_bdT);
-
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s path=%s ia_atime=%s, ia_mtime=%s",
-                                frame->root->unique, uuid_utoa (loc->inode->gfid),
-                                loc->path, actime_str, modtime_str);
+                        ia_time = stbuf->ia_atime;
+                        strftime (actime_str, 256, "[%b %d %H:%M:%S]",
+                                  localtime ((time_t *)&ia_time));
+
+                        ia_time = stbuf->ia_mtime;
+                        strftime (modtime_str, 256, "[%b %d %H:%M:%S]",
+                                  localtime ((time_t *)&ia_time));
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s path=%s "
+                                  "ia_atime=%s, ia_mtime=%s",
+                                  frame->root->unique,
+                                  uuid_utoa (loc->inode->gfid),
+                                  loc->path, actime_str, modtime_str);
+
+                        LOG_ELEMENT (conf, string);
+                        memset (string, 0 , sizeof (string));
                 }
                 frame->local = loc->inode->gfid;
         }
 
+out:
         STACK_WIND (frame, trace_setattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->setattr,
@@ -1695,43 +2061,67 @@ trace_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
         return 0;
 }
 
-
 int
 trace_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
                 struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        char      actime_str[64]  = {0,};
-        char      modtime_str[64] = {0,};
+        uint64_t       ia_time          = 0;
+        char           actime_str[256]  = {0,};
+        char           modtime_str[256] = {0,};
+        trace_conf_t   *conf            = NULL;
 
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FSETATTR].enabled) {
+                char     string[4096]  =  {0,};
                 if (valid & GF_SET_ATTR_MODE) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s fd=%p, mode=%o",
-                                frame->root->unique, uuid_utoa (fd->inode->gfid), fd,
-                                st_mode_from_ia (stbuf->ia_prot, stbuf->ia_type));
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s fd=%p, mode=%o",
+                                  frame->root->unique,
+                                  uuid_utoa (fd->inode->gfid), fd,
+                                  st_mode_from_ia (stbuf->ia_prot,
+                                                   stbuf->ia_type));
+
+                        LOG_ELEMENT (conf, string);
+                        memset (string, 0, sizeof (string));
                 }
 
                 if (valid & (GF_SET_ATTR_UID | GF_SET_ATTR_GID)) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s fd=%p, uid=%o, gid=%o",
-                                frame->root->unique, uuid_utoa (fd->inode->gfid),
-                                fd, stbuf->ia_uid, stbuf->ia_gid);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s fd=%p, uid=%o, "
+                                  "gid=%o", frame->root->unique,
+                                  uuid_utoa (fd->inode->gfid),
+                                  fd, stbuf->ia_uid, stbuf->ia_gid);
+
+                        LOG_ELEMENT (conf, string);
+                        memset (string, 0, sizeof (string));
                 }
 
                 if (valid & (GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME)) {
-                        gf_time_fmt (actime_str, sizeof actime_str,
-                                     stbuf->ia_atime, gf_timefmt_bdT);
-                        gf_time_fmt (modtime_str, sizeof modtime_str,
-                                     stbuf->ia_mtime, gf_timefmt_bdT);
-
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%"PRId64": gfid=%s fd=%p ia_atime=%s, ia_mtime=%s",
-                                frame->root->unique, uuid_utoa (fd->inode->gfid),
-                                fd, actime_str, modtime_str);
+                        ia_time = stbuf->ia_atime;
+                        strftime (actime_str, 256, "[%b %d %H:%M:%S]",
+                                  localtime ((time_t *)&ia_time));
+
+                        ia_time = stbuf->ia_mtime;
+                        strftime (modtime_str, 256, "[%b %d %H:%M:%S]",
+                                  localtime ((time_t *)&ia_time));
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s fd=%p "
+                                  "ia_atime=%s, ia_mtime=%s",
+                                  frame->root->unique,
+                                  uuid_utoa (fd->inode->gfid),
+                                  fd, actime_str, modtime_str);
+
+                        LOG_ELEMENT (conf, string);
+                        memset (string, 0, sizeof (string));
                 }
                 frame->local = fd->inode->gfid;
         }
 
+out:
         STACK_WIND (frame, trace_fsetattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fsetattr,
@@ -1740,19 +2130,30 @@ trace_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
         return 0;
 }
 
-
 int
 trace_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc,
                 off_t offset, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_TRUNCATE].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s path=%s, offset=%"PRId64"",
-                        frame->root->unique, uuid_utoa (loc->inode->gfid),
-                        loc->path, offset);
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s, "
+                          "offset=%"PRId64"", frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          offset);
+
                 frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_truncate_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->truncate,
@@ -1761,19 +2162,30 @@ trace_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc,
         return 0;
 }
 
-
 int
 trace_open (call_frame_t *frame, xlator_t *this, loc_t *loc,
             int32_t flags, fd_t *fd, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_OPEN].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s path=%s flags=%d fd=%p",
-                        frame->root->unique, uuid_utoa (loc->inode->gfid),
-                        loc->path, flags, fd);
+                char      string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s flags=%d fd=%p",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          flags, fd);
+
                 frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_open_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->open,
@@ -1781,42 +2193,62 @@ trace_open (call_frame_t *frame, xlator_t *this, loc_t *loc,
         return 0;
 }
 
-
 int
 trace_create (call_frame_t *frame, xlator_t *this, loc_t *loc,
               int32_t flags, mode_t mode, mode_t umask, fd_t *fd,
               dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_CREATE].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s path=%s fd=%p flags=0%o mode=0%o "
-                        "umask=0%o",
-                        frame->root->unique, uuid_utoa (loc->inode->gfid),
-                        loc->path, fd, flags, mode, umask);
+                char    string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s, fd=%p, "
+                          "flags=0%o mode=0%o umask=0%o",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          fd, flags, mode, umask);
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_create_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->create,
                     loc, flags, mode, umask, fd, xdata);
-
         return 0;
 }
 
-
 int
 trace_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
              size_t size, off_t offset, uint32_t flags, dict_t *xdata)
 {
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_READ].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s fd=%p, size=%"GF_PRI_SIZET", "
-                        "offset=%"PRId64" flags=0%x)",
-                        frame->root->unique, uuid_utoa (fd->inode->gfid),
-                        fd, size, offset, flags);
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s fd=%p, size=%"
+                          GF_PRI_SIZET"offset=%"PRId64" flags=0%x)",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), fd, size,
+                          offset, flags);
+
                 frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_readv_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->readv,
@@ -1824,21 +2256,32 @@ trace_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
         return 0;
 }
 
-
 int
 trace_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
               struct iovec *vector, int32_t count,
               off_t offset, uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
+        trace_conf_t    *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_WRITE].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s fd=%p, count=%d, offset=%"PRId64
-                        " flag=0%x)",
-                        frame->root->unique, uuid_utoa (fd->inode->gfid),
-                        fd, count, offset, flags);
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s fd=%p, count=%d, "
+                          " offset=%"PRId64" flags=0%x)",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), fd, count,
+                          offset, flags);
+
                 frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_writev_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->writev,
@@ -1846,17 +2289,26 @@ trace_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
         return 0;
 }
 
-
 int
 trace_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_STATFS].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s path=%s",
-                        frame->root->unique, (loc->inode)?
-                        uuid_utoa (loc->inode->gfid):"0", loc->path);
+                char  string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s",
+                          frame->root->unique, (loc->inode)?
+                          uuid_utoa (loc->inode->gfid):"0", loc->path);
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_statfs_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->statfs,
@@ -1864,17 +2316,28 @@ trace_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
         return 0;
 }
 
-
 int
 trace_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FLUSH].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s fd=%p",
-                        frame->root->unique, uuid_utoa (fd->inode->gfid), fd);
+                char    string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s fd=%p",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), fd);
+
                 frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_flush_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->flush,
@@ -1882,17 +2345,29 @@ trace_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
         return 0;
 }
 
-
 int
-trace_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags, dict_t *xdata)
+trace_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+             dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FSYNC].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s flags=%d fd=%p",
-                        frame->root->unique, uuid_utoa (fd->inode->gfid), flags, fd);
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s flags=%d fd=%p",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), flags, fd);
+
                 frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_fsync_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fsync,
@@ -1900,19 +2375,30 @@ trace_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags, dict_
         return 0;
 }
 
-
 int
 trace_setxattr (call_frame_t *frame, xlator_t *this,
                 loc_t *loc, dict_t *dict, int32_t flags, dict_t *xdata)
 {
+        trace_conf_t    *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_SETXATTR].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s path=%s flags=%d",
-                        frame->root->unique, uuid_utoa (loc->inode->gfid),
-                        loc->path, flags);
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s flags=%d",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          flags);
+
                 frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_setxattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->setxattr,
@@ -1920,19 +2406,30 @@ trace_setxattr (call_frame_t *frame, xlator_t *this,
         return 0;
 }
 
-
 int
 trace_getxattr (call_frame_t *frame, xlator_t *this,
                 loc_t *loc, const char *name, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_GETXATTR].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s path=%s name=%s",
-                        frame->root->unique, uuid_utoa (loc->inode->gfid),
-                        loc->path, name);
+                char    string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s name=%s",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          name);
+
                 frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_getxattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->getxattr,
@@ -1940,19 +2437,30 @@ trace_getxattr (call_frame_t *frame, xlator_t *this,
         return 0;
 }
 
-
 int
 trace_removexattr (call_frame_t *frame, xlator_t *this,
                    loc_t *loc, const char *name, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_REMOVEXATTR].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s path=%s name=%s",
-                        frame->root->unique, uuid_utoa (loc->inode->gfid),
-                        loc->path, name);
+                char   string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s name=%s",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          name);
+
                 frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_removexattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->removexattr,
@@ -1961,18 +2469,29 @@ trace_removexattr (call_frame_t *frame, xlator_t *this,
         return 0;
 }
 
-
 int
-trace_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd, dict_t *xdata)
+trace_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+               dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_OPENDIR].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s path=%s fd=%p",
-                        frame->root->unique, uuid_utoa (loc->inode->gfid),
-                        loc->path, fd);
+                char   string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s fd=%p",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path, fd);
+
                 frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_opendir_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->opendir,
@@ -1984,15 +2503,27 @@ int
 trace_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
                 off_t offset, dict_t *dict)
 {
+        trace_conf_t *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_READDIRP].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s fd=%p, size=%"GF_PRI_SIZET", "
-                        "offset=%"PRId64" dict=%p",
-                        frame->root->unique, uuid_utoa (fd->inode->gfid),
-                        fd, size, offset, dict);
+                char   string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s fd=%p, size=%"GF_PRI_SIZET
+                          ", offset=%"PRId64" dict=%p",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), fd, size,
+                          offset, dict);
+
                 frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_readdirp_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->readdirp,
@@ -2001,19 +2532,31 @@ trace_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
         return 0;
 }
 
-
 int
 trace_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
                size_t size, off_t offset, dict_t *xdata)
 {
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_READDIR].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s fd=%p, size=%"GF_PRI_SIZET", offset=%"PRId64,
-                        frame->root->unique, uuid_utoa (fd->inode->gfid),
-                        fd, size, offset);
+                char   string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s fd=%p, size=%"GF_PRI_SIZET
+                          ", offset=%"PRId64,
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), fd, size,
+                          offset);
+
                 frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_readdir_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->readdir,
@@ -2022,19 +2565,29 @@ trace_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
         return 0;
 }
 
-
 int
 trace_fsyncdir (call_frame_t *frame, xlator_t *this,
                 fd_t *fd, int32_t datasync, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FSYNCDIR].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s datasync=%d fd=%p",
-                        frame->root->unique, uuid_utoa (fd->inode->gfid),
-                        datasync, fd);
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s datasync=%d fd=%p",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), datasync, fd);
+
                 frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_fsyncdir_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fsyncdir,
@@ -2042,18 +2595,30 @@ trace_fsyncdir (call_frame_t *frame, xlator_t *this,
         return 0;
 }
 
-
 int
-trace_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask, dict_t *xdata)
+trace_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+              dict_t *xdata)
 {
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_ACCESS].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s path=%s mask=0%o",
-                        frame->root->unique, uuid_utoa (loc->inode->gfid),
-                        loc->path, mask);
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s mask=0%o",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid),
+                          loc->path, mask);
+
                 frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_access_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->access,
@@ -2061,19 +2626,30 @@ trace_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask, dic
         return 0;
 }
 
-
 int32_t
 trace_rchecksum (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
                  int32_t len, dict_t *xdata)
 {
+
+        trace_conf_t *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_RCHECKSUM].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s offset=%"PRId64" len=%u fd=%p",
-                        frame->root->unique, uuid_utoa (fd->inode->gfid),
-                        offset, len, fd);
+                char    string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s offset=%"PRId64
+                          "len=%u fd=%p", frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), offset, len, fd);
+
                 frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_rchecksum_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->rchecksum,
@@ -2088,17 +2664,31 @@ trace_fentrylk (call_frame_t *frame, xlator_t *this, const char *volume,
                 fd_t *fd, const char *basename, entrylk_cmd cmd,
                 entrylk_type type, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FENTRYLK].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s volume=%s, (fd=%p basename=%s, "
-                        "cmd=%s, type=%s)",
-                        frame->root->unique, uuid_utoa (fd->inode->gfid),
-                        volume, fd, basename,
-                        ((cmd == ENTRYLK_LOCK) ? "ENTRYLK_LOCK" : "ENTRYLK_UNLOCK"),
-                        ((type == ENTRYLK_RDLCK) ? "ENTRYLK_RDLCK" : "ENTRYLK_WRLCK"));
+                char      string[4096]   =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s volume=%s, (fd=%p "
+                          "basename=%s, cmd=%s, type=%s)",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), volume, fd,
+                          basename,
+                          ((cmd == ENTRYLK_LOCK) ? "ENTRYLK_LOCK" :
+                           "ENTRYLK_UNLOCK"),
+                          ((type == ENTRYLK_RDLCK) ? "ENTRYLK_RDLCK" :
+                           "ENTRYLK_WRLCK"));
+
                 frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_fentrylk_cbk,
                     FIRST_CHILD (this),
                     FIRST_CHILD (this)->fops->fentrylk,
@@ -2111,14 +2701,25 @@ int32_t
 trace_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
                  const char *name, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+                goto out;
         if (trace_fop_names[GF_FOP_FGETXATTR].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s fd=%p name=%s",
-                        frame->root->unique, uuid_utoa (fd->inode->gfid),
-                        fd, name);
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s fd=%p name=%s",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), fd, name);
+
                 frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_fgetxattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fgetxattr,
@@ -2130,14 +2731,25 @@ int32_t
 trace_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
                  dict_t *dict, int32_t flags, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FSETXATTR].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s fd=%p flags=%d",
-                        frame->root->unique, uuid_utoa (fd->inode->gfid),
-                        fd, flags);
+                char   string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s fd=%p flags=%d",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), fd, flags);
+
                 frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_fsetxattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fsetxattr,
@@ -2149,14 +2761,25 @@ int
 trace_ftruncate (call_frame_t *frame, xlator_t *this,
                  fd_t *fd, off_t offset, dict_t *xdata)
 {
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FTRUNCATE].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s offset=%"PRId64" fd=%p",
-                        frame->root->unique, uuid_utoa (fd->inode->gfid),
-                        offset, fd);
+                char    string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s offset=%"PRId64" fd=%p",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), offset, fd);
+
                 frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_ftruncate_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->ftruncate,
@@ -2165,17 +2788,28 @@ trace_ftruncate (call_frame_t *frame, xlator_t *this,
         return 0;
 }
 
-
 int
 trace_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FSTAT].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s fd=%p",
-                        frame->root->unique, uuid_utoa (fd->inode->gfid), fd);
+                char   string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s fd=%p",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), fd);
+
                 frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_fstat_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fstat,
@@ -2183,21 +2817,34 @@ trace_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
         return 0;
 }
 
-
 int
 trace_lk (call_frame_t *frame, xlator_t *this, fd_t *fd,
           int32_t cmd, struct gf_flock *lock, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_LK].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%"PRId64": gfid=%s fd=%p, cmd=%d, lock {l_type=%d, l_whence=%d, "
-                        "l_start=%"PRId64", l_len=%"PRId64", l_pid=%u})",
-                        frame->root->unique, uuid_utoa (fd->inode->gfid), fd,
-                        cmd, lock->l_type, lock->l_whence,
-                        lock->l_start, lock->l_len, lock->l_pid);
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s fd=%p, cmd=%d, "
+                          "lock {l_type=%d, "
+                          "l_whence=%d, l_start=%"PRId64", "
+                          "l_len=%"PRId64", l_pid=%u})",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), fd, cmd,
+                          lock->l_type, lock->l_whence,
+                          lock->l_start, lock->l_len, lock->l_pid);
+
                 frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_lk_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->lk,
@@ -2208,40 +2855,71 @@ trace_lk (call_frame_t *frame, xlator_t *this, fd_t *fd,
 int32_t
 trace_forget (xlator_t *this, inode_t *inode)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
         /* If user want to understand when a lookup happens,
            he should know about 'forget' too */
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_LOOKUP].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "gfid=%s", uuid_utoa (inode->gfid));
+                char   string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "gfid=%s", uuid_utoa (inode->gfid));
+
+                LOG_ELEMENT (conf, string);
         }
+
+out:
         return 0;
 }
 
-
 int32_t
 trace_releasedir (xlator_t *this, fd_t *fd)
 {
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_OPENDIR].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "gfid=%s fd=%p", uuid_utoa (fd->inode->gfid), fd);
+                char   string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "gfid=%s fd=%p",
+                          uuid_utoa (fd->inode->gfid), fd);
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         return 0;
 }
 
 int32_t
 trace_release (xlator_t *this, fd_t *fd)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_OPEN].enabled ||
             trace_fop_names[GF_FOP_CREATE].enabled) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "gfid=%s fd=%p", uuid_utoa (fd->inode->gfid), fd);
+                char   string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "gfid=%s fd=%p",
+                          uuid_utoa (fd->inode->gfid), fd);
+
+                LOG_ELEMENT (conf, string);
         }
+
+out:
         return 0;
 }
 
 
-
 void
 enable_all_calls (int enabled)
 {
@@ -2251,7 +2929,6 @@ enable_all_calls (int enabled)
                 trace_fop_names[i].enabled = enabled;
 }
 
-
 void
 enable_call (const char *name, int enabled)
 {
@@ -2279,6 +2956,105 @@ process_call_list (const char *list, int include)
         }
 }
 
+int32_t
+trace_dump_history (xlator_t *this)
+{
+        int ret = -1;
+        char key_prefix[GF_DUMP_MAX_BUF_LEN] = {0,};
+        trace_conf_t *conf = NULL;
+
+        GF_VALIDATE_OR_GOTO ("trace", this, out);
+        GF_VALIDATE_OR_GOTO (this->name, this->history, out);
+
+        conf = this->private;
+        // Is it ok to return silently if log-history option his off?
+        if (conf && conf->log_history == _gf_true) {
+                gf_proc_dump_build_key (key_prefix, "xlator.debug.trace",
+                                        "history");
+                gf_proc_dump_add_section (key_prefix);
+                eh_dump (this->history, NULL, dump_history_trace);
+        }
+        ret = 0;
+
+out:
+        return ret;
+}
+
+int32_t
+mem_acct_init (xlator_t *this)
+{
+        int     ret = -1;
+
+        if (!this)
+                return ret;
+
+        ret = xlator_mem_acct_init (this, gf_trace_mt_end + 1);
+
+        if (ret != 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
+                        " failed");
+                return ret;
+        }
+
+        return ret;
+}
+
+int
+reconfigure (xlator_t *this, dict_t *options)
+{
+        int32_t       ret            = -1;
+        trace_conf_t    *conf           = NULL;
+        char            *includes = NULL, *excludes = NULL;
+
+        GF_VALIDATE_OR_GOTO ("quick-read", this, out);
+        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
+        GF_VALIDATE_OR_GOTO (this->name, options, out);
+
+        conf = this->private;
+
+        includes = data_to_str (dict_get (options, "include-ops"));
+        excludes = data_to_str (dict_get (options, "exclude-ops"));
+
+        {
+                int i;
+                for (i = 0; i < GF_FOP_MAXVALUE; i++) {
+                        if (gf_fop_list[i])
+                                strncpy (trace_fop_names[i].name,
+                                         gf_fop_list[i],
+                                         strlen (gf_fop_list[i]));
+                        else
+                                strncpy (trace_fop_names[i].name, ":O",
+                                         strlen (":O"));
+                        trace_fop_names[i].enabled = 1;
+                }
+        }
+
+        if (includes && excludes) {
+                gf_log (this->name,
+                        GF_LOG_ERROR,
+                        "must specify only one of 'include-ops' and "
+                        "'exclude-ops'");
+                goto out;
+        }
+
+        if (includes)
+                process_call_list (includes, 1);
+        if (excludes)
+                process_call_list (excludes, 0);
+
+        /* Should resizing of the event-history be allowed in reconfigure?
+         * for which a new event_history might have to be allocated and the
+         * older history has to be freed.
+         */
+        GF_OPTION_RECONF ("log-file", conf->log_file, options, bool, out);
+
+        GF_OPTION_RECONF ("log-history", conf->log_history, options, bool, out);
+
+        ret = 0;
+
+out:
+        return ret;
+}
 
 int32_t
 init (xlator_t *this)
@@ -2286,6 +3062,10 @@ init (xlator_t *this)
         dict_t *options = NULL;
         char *includes = NULL, *excludes = NULL;
         char *forced_loglevel = NULL;
+        eh_t *history = NULL;
+        int  ret = -1;
+        size_t  history_size = TRACE_DEFAULT_HISTORY_SIZE;
+        trace_conf_t    *conf = NULL;
 
         if (!this)
                 return -1;
@@ -2300,6 +3080,12 @@ init (xlator_t *this)
                         "dangling volume. check volfile ");
         }
 
+        conf = GF_CALLOC (1, sizeof (trace_conf_t), gf_trace_mt_trace_conf_t);
+        if (!conf) {
+                gf_log (this->name, GF_LOG_ERROR, "cannot allocate "
+                        "xl->private");
+                return -1;
+        }
 
         options = this->options;
         includes = data_to_str (dict_get (options, "include-ops"));
@@ -2308,8 +3094,13 @@ init (xlator_t *this)
         {
                 int i;
                 for (i = 0; i < GF_FOP_MAXVALUE; i++) {
-                        trace_fop_names[i].name = (gf_fop_list[i] ?
-                                                   gf_fop_list[i] : ":O");
+                        if (gf_fop_list[i])
+                                strncpy (trace_fop_names[i].name,
+                                         gf_fop_list[i],
+                                         strlen (gf_fop_list[i]));
+                        else
+                                strncpy (trace_fop_names[i].name, ":O",
+                                         strlen (":O"));
                         trace_fop_names[i].enabled = 1;
                 }
         }
@@ -2317,14 +3108,43 @@ init (xlator_t *this)
         if (includes && excludes) {
                 gf_log (this->name,
                         GF_LOG_ERROR,
-                        "must specify only one of 'include-ops' and 'exclude-ops'");
+                        "must specify only one of 'include-ops' and "
+                        "'exclude-ops'");
                 return -1;
         }
+
         if (includes)
                 process_call_list (includes, 1);
         if (excludes)
                 process_call_list (excludes, 0);
 
+
+        GF_OPTION_INIT ("history-size", conf->history_size, size, out);
+
+        gf_log (this->name, GF_LOG_INFO, "history size %"GF_PRI_SIZET,
+                history_size);
+
+        GF_OPTION_INIT ("log-file", conf->log_file, bool, out);
+
+        gf_log (this->name, GF_LOG_INFO, "logging to file %s",
+                (conf->log_file == _gf_true)?"enabled":"disabled");
+
+        GF_OPTION_INIT ("log-history", conf->log_history, bool, out);
+
+        gf_log (this->name, GF_LOG_DEBUG, "logging to history %s",
+                (conf->log_history == _gf_true)?"enabled":"disabled");
+
+        history = eh_new (history_size, _gf_false, NULL);
+        if (!history) {
+                gf_log (this->name, GF_LOG_ERROR, "event history cannot be "
+                        "initialized");
+                return -1;
+        }
+
+        this->history = history;
+
+        conf->trace_log_level = GF_LOG_INFO;
+
         if (dict_get (options, "force-log-level")) {
                 forced_loglevel = data_to_str (dict_get (options,
                                                          "force-log-level"));
@@ -2332,25 +3152,34 @@ init (xlator_t *this)
                         goto setloglevel;
 
                 if (strcmp (forced_loglevel, "INFO") == 0)
-                        trace_log_level = GF_LOG_INFO;
+                        conf->trace_log_level = GF_LOG_INFO;
                 else if (strcmp (forced_loglevel, "TRACE") == 0)
-                        trace_log_level = GF_LOG_TRACE;
+                        conf->trace_log_level = GF_LOG_TRACE;
                 else if (strcmp (forced_loglevel, "ERROR") == 0)
-                        trace_log_level = GF_LOG_ERROR;
+                        conf->trace_log_level = GF_LOG_ERROR;
                 else if (strcmp (forced_loglevel, "DEBUG") == 0)
-                        trace_log_level = GF_LOG_DEBUG;
+                        conf->trace_log_level = GF_LOG_DEBUG;
                 else if (strcmp (forced_loglevel, "WARNING") == 0)
-                        trace_log_level = GF_LOG_WARNING;
+                        conf->trace_log_level = GF_LOG_WARNING;
                 else if (strcmp (forced_loglevel, "CRITICAL") == 0)
-                        trace_log_level = GF_LOG_CRITICAL;
+                        conf->trace_log_level = GF_LOG_CRITICAL;
                 else if (strcmp (forced_loglevel, "NONE") == 0)
-                        trace_log_level = GF_LOG_NONE;
+                        conf->trace_log_level = GF_LOG_NONE;
         }
 
 setloglevel:
-        gf_log_set_loglevel (trace_log_level);
+        gf_log_set_loglevel (conf->trace_log_level);
+        this->private = conf;
+        ret = 0;
+out:
+        if (ret == -1) {
+                if (history)
+                        GF_FREE (history);
+                if (conf)
+                        GF_FREE (conf);
+        }
 
-        return 0;
+        return ret;
 }
 
 void
@@ -2359,6 +3188,9 @@ fini (xlator_t *this)
         if (!this)
                 return;
 
+        if (this->history)
+                eh_destroy (this->history);
+
         gf_log (this->name, GF_LOG_INFO,
                 "trace translator unloaded");
         return;
@@ -2407,7 +3239,6 @@ struct xlator_fops fops = {
         .fsetattr    = trace_fsetattr,
 };
 
-
 struct xlator_cbks cbks = {
         .release     = trace_release,
         .releasedir  = trace_releasedir,
@@ -2423,5 +3254,21 @@ struct volume_options options[] = {
           .type = GF_OPTION_TYPE_STR
           /*.value = { ""} */
         },
+        { .key  = {"history-size"},
+          .type = GF_OPTION_TYPE_SIZET,
+          .default_value = "1024",
+        },
+        { .key  = {"log-file"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "no",
+        },
+        { .key  = {"log-history"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "no",
+        },
         { .key  = {NULL} },
 };
+
+struct xlator_dumpops dumpops = {
+        .history = trace_dump_history
+};
diff --git a/xlators/debug/trace/src/trace.h b/xlators/debug/trace/src/trace.h
new file mode 100644
index 000000000..045eefb36
--- /dev/null
+++ b/xlators/debug/trace/src/trace.h
@@ -0,0 +1,98 @@
+/*
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include <time.h>
+#include <errno.h>
+#include "glusterfs.h"
+#include "xlator.h"
+#include "common-utils.h"
+#include "event-history.h"
+#include "logging.h"
+#include "circ-buff.h"
+#include "statedump.h"
+#include "options.h"
+
+#define TRACE_DEFAULT_HISTORY_SIZE 1024
+
+typedef struct {
+        /* Since the longest fop name is fremovexattr i.e 12 characters, array size
+         * is kept 24, i.e double of the maximum.
+         */
+        char name[24];
+        int enabled;
+} trace_fop_name_t;
+
+trace_fop_name_t trace_fop_names[GF_FOP_MAXVALUE];
+
+typedef struct {
+        gf_boolean_t log_file;
+        gf_boolean_t log_history;
+        size_t       history_size;
+        int trace_log_level;
+} trace_conf_t;
+
+#define TRACE_STACK_UNWIND(op, frame, params ...)                       \
+        do {                                                            \
+                frame->local = NULL;                                    \
+                STACK_UNWIND_STRICT (op, frame, params);                \
+        } while (0);
+
+#define LOG_ELEMENT(_conf, _string)                                     \
+        do {                                                            \
+                if (_conf) {                                            \
+                        if ((_conf->log_history) == _gf_true)           \
+                                gf_log_eh ("%s", _string);              \
+                        if ((_conf->log_file) == _gf_true)              \
+                                gf_log (THIS->name, _conf->trace_log_level, \
+                                        "%s", _string);                 \
+                }                                                       \
+        } while (0);
+
+#define trace_stat_to_str(buf, statstr)                                 \
+        do {                                                            \
+                char     atime_buf[256]    = {0,};                      \
+                char     mtime_buf[256]    = {0,};                      \
+                char     ctime_buf[256]    = {0,};                      \
+                uint64_t ia_time           = 0;                         \
+                                                                        \
+                if (!buf)                                               \
+                        break;                                          \
+                                                                        \
+                ia_time = buf->ia_atime;                                \
+                strftime (atime_buf, 256, "[%b %d %H:%M:%S]",           \
+                          localtime ((time_t *)&ia_time));              \
+                                                                        \
+                ia_time = buf->ia_mtime;                                \
+                strftime (mtime_buf, 256, "[%b %d %H:%M:%S]",           \
+                          localtime ((time_t *)&ia_time));              \
+                                                                        \
+                ia_time = buf->ia_ctime;                                \
+                strftime (ctime_buf, 256, "[%b %d %H:%M:%S]",           \
+                          localtime ((time_t *)&ia_time));              \
+                                                                        \
+                snprintf (statstr, sizeof (statstr),                    \
+                          "gfid=%s ino=%"PRIu64", mode=%o, "            \
+                          "nlink=%"GF_PRI_NLINK", uid=%u, "             \
+                          "gid=%u, size=%"PRIu64", "                    \
+                          "blocks=%"PRIu64", atime=%s, "                \
+                          "mtime=%s, ctime=%s",                         \
+                          uuid_utoa (buf->ia_gfid), buf->ia_ino,        \
+                          st_mode_from_ia (buf->ia_prot,                \
+                                           buf->ia_type),               \
+                          buf->ia_nlink, buf->ia_uid,                   \
+                          buf->ia_gid, buf->ia_size,                    \
+                          buf->ia_blocks, atime_buf,                    \
+                          mtime_buf, ctime_buf);                        \
+        } while (0);
diff --git a/xlators/encryption/Makefile.am b/xlators/encryption/Makefile.am
index 2cbde680f..36efc6698 100644
--- a/xlators/encryption/Makefile.am
+++ b/xlators/encryption/Makefile.am
@@ -1,3 +1,3 @@
-SUBDIRS = rot-13
+SUBDIRS = rot-13 crypt
 
 CLEANFILES = 
diff --git a/xlators/cluster/unify/Makefile.am b/xlators/encryption/crypt/Makefile.am
index d471a3f92..d471a3f92 100644
--- a/xlators/cluster/unify/Makefile.am
+++ b/xlators/encryption/crypt/Makefile.am
diff --git a/xlators/encryption/crypt/src/Makefile.am b/xlators/encryption/crypt/src/Makefile.am
new file mode 100644
index 000000000..b13f65043
--- /dev/null
+++ b/xlators/encryption/crypt/src/Makefile.am
@@ -0,0 +1,24 @@
+if ENABLE_CRYPT_XLATOR
+
+xlator_LTLIBRARIES = crypt.la
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/encryption
+
+crypt_la_LDFLAGS = -module -avoid-version -lssl -lcrypto
+
+crypt_la_SOURCES = keys.c data.c metadata.c atom.c crypt.c
+crypt_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+noinst_HEADERS = crypt-common.h crypt-mem-types.h crypt.h metadata.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+CLEANFILES =
+
+else
+
+noinst_DIST = keys.c data.c metadata.c atom.c crypt.c
+noinst_HEADERS = crypt-common.h crypt-mem-types.h crypt.h metadata.h
+
+endif
+\ No newline at end of file
diff --git a/xlators/encryption/crypt/src/atom.c b/xlators/encryption/crypt/src/atom.c
new file mode 100644
index 000000000..1ec41495c
--- /dev/null
+++ b/xlators/encryption/crypt/src/atom.c
@@ -0,0 +1,962 @@
+/*
+  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "defaults.h"
+#include "crypt-common.h"
+#include "crypt.h"
+
+/*
+ *                              Glossary
+ *
+ *
+ * cblock            (or cipher block). A logical unit in a file.
+ *                   cblock size is defined as the number of bits
+ *                   in an input (or output) block of the block
+ *                   cipher (*). Cipher block size is a property of
+ *                   cipher algorithm. E.g. cblock size is 64 bits
+ *                   for DES, 128 bits for AES, etc.
+ *
+ * atomic cipher     A cipher algorithm, which requires some chunks of
+ * algorithm         text to be padded at left and(or) right sides before
+ *                   cipher transaform.
+ *
+ *
+ * block (atom)      Minimal chunk of file's data, which doesn't require
+ *                   padding. We'll consider logical units in a file of
+ *                   block size (atom size).
+ *
+ * cipher algorithm  Atomic cipher algorithm, which requires the last
+ * with EOF issue    incomplete cblock in a file to be padded with some
+ *                   data (usually zeros).
+ *
+ *
+ * operation, which  reading/writing from offset, which is not aligned to
+ * forms a gap at    to atom size
+ * the beginning
+ *
+ *
+ * operation, which  reading/writing count bytes starting from offset off,
+ * forms a gap at    so that off+count is not aligned to atom_size
+ * the end
+ *
+ * head block        the first atom affected by an operation, which forms
+ *                   a gap at the beginning, or(and) at the end.
+ *                   Сomment. Head block has at least one gap (either at
+ *                   the beginning, or at the end)
+ *
+ *
+ * tail block        the last atom different from head, affected by an
+ *                   operation, which forms a gap at the end.
+ *                   Сomment: Tail block has exactly one gap (at the end).
+ *
+ *
+ * partial block     head or tail block
+ *
+ *
+ * full block        block without gaps.
+ *
+ *
+ * (*) Recommendation for Block Cipher Modes of Operation
+ *     Methods and Techniques
+ *     NIST Special Publication 800-38A Edition 2001
+ */
+
+/*
+ * atom->offset_at()
+ */
+static off_t offset_at_head(struct avec_config *conf)
+{
+	return conf->aligned_offset;
+}
+
+static off_t offset_at_hole_head(call_frame_t *frame,
+					 struct object_cipher_info *object)
+{
+	return offset_at_head(get_hole_conf(frame));
+}
+
+static off_t offset_at_data_head(call_frame_t *frame,
+					 struct object_cipher_info *object)
+{
+	return offset_at_head(get_data_conf(frame));
+}
+
+
+static off_t offset_at_tail(struct avec_config *conf,
+				    struct object_cipher_info *object)
+{
+	return conf->aligned_offset +
+		(conf->off_in_head ? get_atom_size(object) : 0) +
+		(conf->nr_full_blocks << get_atom_bits(object));
+}
+
+static off_t offset_at_hole_tail(call_frame_t *frame,
+					 struct object_cipher_info *object)
+{
+	return offset_at_tail(get_hole_conf(frame), object);
+}
+
+
+static off_t offset_at_data_tail(call_frame_t *frame,
+					 struct object_cipher_info *object)
+{
+	return offset_at_tail(get_data_conf(frame), object);
+}
+
+static off_t offset_at_full(struct avec_config *conf,
+				    struct object_cipher_info *object)
+{
+	return conf->aligned_offset +
+		(conf->off_in_head ? get_atom_size(object) : 0);
+}
+
+static off_t offset_at_data_full(call_frame_t *frame,
+					 struct object_cipher_info *object)
+{
+	return offset_at_full(get_data_conf(frame), object);
+}
+
+static off_t offset_at_hole_full(call_frame_t *frame,
+					 struct object_cipher_info *object)
+{
+	return offset_at_full(get_hole_conf(frame), object);
+}
+
+/*
+ * atom->io_size_nopad()
+ */
+
+static uint32_t io_size_nopad_head(struct avec_config *conf,
+				   struct object_cipher_info *object)
+{
+	uint32_t gap_at_beg;
+	uint32_t gap_at_end;
+
+	check_head_block(conf);
+
+	gap_at_beg = conf->off_in_head;
+
+	if (has_tail_block(conf) || has_full_blocks(conf) || conf->off_in_tail == 0 )
+		gap_at_end = 0;
+	else
+		gap_at_end = get_atom_size(object) - conf->off_in_tail;
+
+	return get_atom_size(object) - (gap_at_beg + gap_at_end);
+}
+
+static uint32_t io_size_nopad_tail(struct avec_config *conf,
+				   struct object_cipher_info *object)
+{
+	check_tail_block(conf);
+	return conf->off_in_tail;
+}
+
+static uint32_t io_size_nopad_full(struct avec_config *conf,
+				   struct object_cipher_info *object)
+{	
+	check_full_block(conf);
+	return get_atom_size(object);
+}
+
+static uint32_t io_size_nopad_data_head(call_frame_t *frame,
+					struct object_cipher_info *object)
+{
+	return io_size_nopad_head(get_data_conf(frame), object);
+}
+
+static uint32_t io_size_nopad_hole_head(call_frame_t *frame,
+					struct object_cipher_info *object)
+{
+	return io_size_nopad_head(get_hole_conf(frame), object);
+}
+
+static uint32_t io_size_nopad_data_tail(call_frame_t *frame,
+					struct object_cipher_info *object)
+{
+	return io_size_nopad_tail(get_data_conf(frame), object);
+}
+
+static uint32_t io_size_nopad_hole_tail(call_frame_t *frame,
+					struct object_cipher_info *object)
+{
+	return io_size_nopad_tail(get_hole_conf(frame), object);
+}
+
+static uint32_t io_size_nopad_data_full(call_frame_t *frame,
+					struct object_cipher_info *object)
+{
+	return io_size_nopad_full(get_data_conf(frame), object);
+}
+
+static uint32_t io_size_nopad_hole_full(call_frame_t *frame,
+					struct object_cipher_info *object)
+{
+	return io_size_nopad_full(get_hole_conf(frame), object);
+}
+
+static uint32_t offset_in_head(struct avec_config *conf)
+{
+	check_cursor_head(conf);
+
+	return conf->off_in_head;
+}
+
+static uint32_t offset_in_tail(call_frame_t *frame,
+			       struct object_cipher_info *object)
+{
+	return 0;
+}
+
+static uint32_t offset_in_full(struct avec_config *conf,
+			       struct object_cipher_info *object)
+{
+	check_cursor_full(conf);
+
+	if (has_head_block(conf))
+		return (conf->cursor - 1) << get_atom_bits(object);
+	else
+		return conf->cursor << get_atom_bits(object);
+}
+
+static uint32_t offset_in_data_head(call_frame_t *frame,
+				    struct object_cipher_info *object)
+{
+	return offset_in_head(get_data_conf(frame));
+}
+
+static uint32_t offset_in_hole_head(call_frame_t *frame,
+				    struct object_cipher_info *object)
+{
+	return offset_in_head(get_hole_conf(frame));
+}
+
+static uint32_t offset_in_data_full(call_frame_t *frame,
+				    struct object_cipher_info *object)
+{
+	return offset_in_full(get_data_conf(frame), object);
+}
+
+static uint32_t offset_in_hole_full(call_frame_t *frame,
+				    struct object_cipher_info *object)
+{
+	return offset_in_full(get_hole_conf(frame), object);
+}
+
+/*
+ * atom->rmw()
+ */
+/*
+ * Pre-conditions:
+ * @vec contains plain text of the latest
+ * version.
+ *
+ * Uptodate gaps of the @partial block with
+ * this plain text, encrypt the whole block
+ * and write the result to disk.
+ */
+static int32_t rmw_partial_block(call_frame_t *frame,
+				 void *cookie,
+				 xlator_t *this,
+				 int32_t op_ret,
+				 int32_t op_errno,
+				 struct iovec *vec,
+				 int32_t count,
+				 struct iatt *stbuf,
+				 struct iobref *iobref,
+				 struct rmw_atom *atom)
+{
+	size_t was_read = 0;
+	uint64_t file_size;
+	crypt_local_t *local = frame->local;
+	struct object_cipher_info *object = &local->info->cinfo;
+
+	struct iovec *partial = atom->get_iovec(frame, 0);
+	struct avec_config *conf = atom->get_config(frame);
+	end_writeback_handler_t end_writeback_partial_block;
+#if DEBUG_CRYPT
+	gf_boolean_t check_last_cblock = _gf_false;
+#endif
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (op_ret < 0)
+		goto exit;
+
+	file_size = local->cur_file_size;
+	was_read = op_ret;
+
+	if (atom->locality == HEAD_ATOM && conf->off_in_head) {
+		/*
+		 * head atom with a non-uptodate gap
+		 * at the beginning
+		 *
+		 * fill the gap with plain text of the
+		 * latest version. Convert a part of hole
+		 * (if any) to zeros.
+		 */
+		int32_t i;
+		int32_t copied = 0;
+		int32_t to_gap; /* amount of data needed to uptodate
+				   the gap at the beginning */
+#if 0
+		int32_t hole = 0; /* The part of the hole which
+				   * got in the head block */
+#endif /* 0 */
+		to_gap = conf->off_in_head;
+
+		if (was_read < to_gap) {
+			if (file_size >
+			    offset_at_head(conf) + was_read) {
+				/*
+				 * It is impossible to uptodate
+				 * head block: too few bytes have
+				 * been read from disk, so that
+				 * partial write is impossible.
+				 *
+				 * It could happen because of many
+				 * reasons: IO errors, (meta)data
+				 * corruption in the local file system,
+				 * etc.
+				 */
+				gf_log(this->name, GF_LOG_WARNING,
+				     "Can not uptodate a gap at the beginning");
+				local->op_ret = -1;
+				local->op_errno = EIO;
+				goto exit;
+			}
+#if 0
+			hole = to_gap - was_read;
+#endif /* 0 */
+			to_gap = was_read;
+		}
+		/*
+		 * uptodate the gap at the beginning
+		 */
+		for (i = 0; i < count && copied < to_gap; i++) {
+			int32_t to_copy;
+
+			to_copy = vec[i].iov_len;
+			if (to_copy > to_gap - copied)
+				to_copy = to_gap - copied;
+
+			memcpy(partial->iov_base, vec[i].iov_base, to_copy);
+			copied += to_copy;
+		}
+#if 0
+		/*
+		 * If possible, convert part of the
+		 * hole, which got in the head block
+		 */
+		ret = TRY_LOCK(&local->hole_lock);
+		if (!ret) {
+			if (local->hole_handled)
+				/*
+				 * already converted by
+				 * crypt_writev_cbk()
+				 */
+				UNLOCK(&local->hole_lock);
+			else {
+				/*
+				 * convert the part of the hole
+				 * which got in the head block
+				 * to zeros.
+				 *
+				 * Update the orig_offset to make
+				 * sure writev_cbk() won't care
+				 * about this part of the hole.
+				 *
+				 */
+				memset(partial->iov_base + to_gap, 0, hole);
+
+				conf->orig_offset -= hole;
+				conf->orig_size += hole;
+				UNLOCK(&local->hole_lock);
+			}
+		}
+		else /*
+		      * conversion is being performed
+		      * by crypt_writev_cbk()
+		      */
+			;
+#endif /* 0 */
+	}
+	if (atom->locality == TAIL_ATOM ||
+	    (!has_tail_block(conf) && conf->off_in_tail)) {
+		/*
+		 * tail atom, or head atom with a non-uptodate
+		 * gap at the end.
+		 *
+		 * fill the gap at the end of the block
+		 * with plain text of the latest version.
+		 * Pad the result, (if needed)
+		 */
+		int32_t i;
+		int32_t to_gap;
+		int copied;
+		off_t off_in_tail;
+		int32_t to_copy;
+
+		off_in_tail = conf->off_in_tail;
+		to_gap = conf->gap_in_tail;
+
+		if (to_gap && was_read < off_in_tail + to_gap) {
+			/*
+			 * It is impossible to uptodate
+			 * the gap at the end: too few bytes
+			 * have been read from disk, so that
+			 * partial write is impossible.
+			 *
+			 * It could happen because of many
+			 * reasons: IO errors, (meta)data
+			 * corruption in the local file system,
+			 * etc.
+			 */
+			gf_log(this->name, GF_LOG_WARNING,
+			       "Can not uptodate a gap at the end");
+			local->op_ret = -1;
+			local->op_errno = EIO;
+			goto exit;
+		}
+		/*
+		 * uptodate the gap at the end
+		 */
+		copied = 0;
+		to_copy = to_gap;
+		for(i = count - 1; i >= 0 && to_copy > 0; i--) {
+			uint32_t from_vec, off_in_vec;
+
+			off_in_vec = 0;
+			from_vec = vec[i].iov_len;
+			if (from_vec > to_copy) {
+				off_in_vec = from_vec - to_copy;
+				from_vec = to_copy;
+			}
+			memcpy(partial->iov_base +
+			       off_in_tail + to_gap - copied - from_vec,
+			       vec[i].iov_base + off_in_vec,
+			       from_vec);
+
+			gf_log(this->name, GF_LOG_DEBUG,
+			       "uptodate %d bytes at tail. Offset at target(source): %d(%d)",
+			       (int)from_vec,
+			       (int)off_in_tail + to_gap - copied - from_vec,
+			       (int)off_in_vec);
+
+			copied += from_vec;
+			to_copy -= from_vec;
+		}
+		partial->iov_len = off_in_tail + to_gap;
+
+		if (object_alg_should_pad(object)) {
+			int32_t resid = 0;
+			resid = partial->iov_len & (object_alg_blksize(object) - 1);
+			if (resid) {
+				/*
+				 * append a new EOF padding
+				 */
+				local->eof_padding_size =
+					object_alg_blksize(object) - resid;
+
+				gf_log(this->name, GF_LOG_DEBUG,
+				       "set padding size %d",
+				       local->eof_padding_size);
+
+				memset(partial->iov_base + partial->iov_len,
+				       1,
+				       local->eof_padding_size);
+				partial->iov_len += local->eof_padding_size;
+#if DEBUG_CRYPT
+				gf_log(this->name, GF_LOG_DEBUG,
+				       "pad cblock with %d zeros:",
+				       local->eof_padding_size);
+				dump_cblock(this,
+					    (unsigned char *)partial->iov_base +
+					    partial->iov_len - object_alg_blksize(object));
+				check_last_cblock = _gf_true;
+#endif
+			}
+		}
+	}
+	/*
+	 * encrypt the whole block
+	 */
+	encrypt_aligned_iov(object,
+			    partial,
+			    1,
+			    atom->offset_at(frame, object));
+#if DEBUG_CRYPT
+	if (check_last_cblock == _gf_true) {
+		gf_log(this->name, GF_LOG_DEBUG,
+		       "encrypt last cblock with offset %llu",
+		       (unsigned long long)atom->offset_at(frame, object));
+		dump_cblock(this, (unsigned char *)partial->iov_base +
+			    partial->iov_len - object_alg_blksize(object));
+	}
+#endif
+	set_local_io_params_writev(frame, object, atom,
+				   atom->offset_at(frame, object),
+				   iovec_get_size(partial, 1));
+	/*
+	 * write the whole block to disk
+	 */
+	end_writeback_partial_block = dispatch_end_writeback(local->fop);
+	conf->cursor ++;
+	STACK_WIND(frame,
+		   end_writeback_partial_block,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->writev,
+		   local->fd,
+		   partial,
+		   1,
+		   atom->offset_at(frame, object),
+		   local->flags,
+		   local->iobref_data,
+		   local->xdata);
+
+	gf_log("crypt", GF_LOG_DEBUG,
+	       "submit partial block: %d bytes from %d offset",
+	       (int)iovec_get_size(partial, 1),
+	       (int)atom->offset_at(frame, object));
+ exit:
+	return 0;
+}
+
+/*
+ * Perform a (read-)modify-write sequence.
+ * This should be performed only after approval
+ * of upper server-side manager, i.e. the caller
+ * needs to make sure this is his turn to rmw.
+ */
+void submit_partial(call_frame_t *frame,
+		    xlator_t *this,
+		    fd_t *fd,
+		    atom_locality_type ltype)
+{
+	int32_t ret;
+	dict_t *dict;
+	struct rmw_atom *atom;
+	crypt_local_t *local = frame->local;
+	struct object_cipher_info *object = &local->info->cinfo;
+
+	atom = atom_by_types(local->active_setup, ltype);
+	/*
+	 * To perform the "read" component of the read-modify-write
+	 * sequence the crypt translator does stack_wind to itself.
+	 * 
+	 * Pass current file size to crypt_readv()
+	 */
+	dict = dict_new();
+	if (!dict) {
+		/*
+		 * FIXME: Handle the error
+		 */
+		gf_log("crypt", GF_LOG_WARNING, "Can not alloc dict");
+		return;
+	}
+	ret = dict_set(dict,
+		       FSIZE_XATTR_PREFIX,
+		       data_from_uint64(local->cur_file_size));
+	if (ret) {
+		/*
+		 * FIXME: Handle the error
+		 */
+		dict_unref(dict);
+		gf_log("crypt", GF_LOG_WARNING, "Can not set dict");
+		goto exit;
+	}
+	STACK_WIND(frame,
+		   atom->rmw,
+		   this,
+		   this->fops->readv, /* crypt_readv */
+		   fd,
+		   atom->count_to_uptodate(frame, object), /* count */
+		   atom->offset_at(frame, object), /* offset to read from */
+		   0,
+		   dict);
+ exit:
+	dict_unref(dict);
+}
+
+/*
+ * submit blocks of FULL_ATOM type
+ */
+void submit_full(call_frame_t *frame, xlator_t *this)
+{
+	crypt_local_t *local = frame->local;
+	struct object_cipher_info *object = &local->info->cinfo;
+	struct rmw_atom *atom = atom_by_types(local->active_setup, FULL_ATOM);
+	uint32_t count; /* total number of full blocks to submit */
+	uint32_t granularity; /* number of blocks to submit in one iteration */
+
+	uint64_t off_in_file; /* start offset in the file, bytes */
+	uint32_t off_in_atom; /* start offset in the atom, blocks */
+	uint32_t blocks_written = 0; /* blocks written for this submit */
+
+	struct avec_config *conf = atom->get_config(frame);
+	end_writeback_handler_t end_writeback_full_block;
+	/*
+	 * Write full blocks by groups of granularity size.
+	 */
+	end_writeback_full_block = dispatch_end_writeback(local->fop);
+
+	if (is_ordered_mode(frame)) {
+		uint32_t skip = has_head_block(conf) ? 1 : 0;
+		count = 1;
+		granularity = 1;
+		/*
+		 * calculate start offset using cursor value;
+		 * here we should take into accout head block,
+		 * which corresponds to cursor value 0.
+		 */
+		off_in_file = atom->offset_at(frame, object) +
+			((conf->cursor - skip) << get_atom_bits(object));
+		off_in_atom = conf->cursor - skip;
+	}
+	else {
+		/*
+		 * in parallel mode
+		 */
+		count = conf->nr_full_blocks;
+		granularity = MAX_IOVEC;
+		off_in_file = atom->offset_at(frame, object);
+		off_in_atom = 0;
+	}
+	while (count) {
+		uint32_t blocks_to_write = count;
+
+		if (blocks_to_write > granularity)
+			blocks_to_write = granularity;
+		if (conf->type == HOLE_ATOM)
+			/*
+			 * reset iovec before encryption
+			 */
+			memset(atom->get_iovec(frame, 0)->iov_base,
+			       0,
+			       get_atom_size(object));
+		/*
+		 * encrypt the group
+		 */
+		encrypt_aligned_iov(object,
+				    atom->get_iovec(frame,
+						    off_in_atom +
+						    blocks_written),
+				    blocks_to_write,
+				    off_in_file + (blocks_written <<
+						   get_atom_bits(object)));
+		
+		set_local_io_params_writev(frame, object, atom,
+		        off_in_file + (blocks_written << get_atom_bits(object)),
+			blocks_to_write <<  get_atom_bits(object));
+
+		conf->cursor += blocks_to_write;
+
+		STACK_WIND(frame,
+			   end_writeback_full_block,
+			   FIRST_CHILD(this),
+			   FIRST_CHILD(this)->fops->writev,
+			   local->fd,
+			   atom->get_iovec(frame, off_in_atom + blocks_written),
+			   blocks_to_write,
+			   off_in_file + (blocks_written << get_atom_bits(object)),
+			   local->flags,
+			   local->iobref_data ? local->iobref_data : local->iobref,
+			   local->xdata);
+
+		gf_log("crypt", GF_LOG_DEBUG, "submit %d full blocks from %d offset",
+		       blocks_to_write,
+		       (int)(off_in_file + (blocks_written << get_atom_bits(object))));
+
+		count -= blocks_to_write;
+		blocks_written += blocks_to_write;
+	}
+	return;
+}
+
+static int32_t rmw_data_head(call_frame_t *frame,
+			     void *cookie,
+			     xlator_t *this,
+			     int32_t op_ret,
+			     int32_t op_errno,
+			     struct iovec *vec,
+			     int32_t count,
+			     struct iatt *stbuf,
+			     struct iobref *iobref,
+			     dict_t *xdata)
+{
+	return rmw_partial_block(frame,
+				 cookie,
+				 this,
+				 op_ret,
+				 op_errno,
+				 vec,
+				 count,
+				 stbuf,
+				 iobref,
+				 atom_by_types(DATA_ATOM, HEAD_ATOM));
+}
+
+static int32_t rmw_data_tail(call_frame_t *frame,
+			     void *cookie,
+			     xlator_t *this,
+			     int32_t op_ret,
+			     int32_t op_errno,
+			     struct iovec *vec,
+			     int32_t count,
+			     struct iatt *stbuf,
+			     struct iobref *iobref,
+			     dict_t *xdata)
+{
+	return rmw_partial_block(frame,
+				 cookie,
+				 this,
+				 op_ret,
+				 op_errno,
+				 vec,
+				 count,
+				 stbuf,
+				 iobref,
+				 atom_by_types(DATA_ATOM, TAIL_ATOM));
+}
+
+static int32_t rmw_hole_head(call_frame_t *frame,
+			     void *cookie,
+			     xlator_t *this,
+			     int32_t op_ret,
+			     int32_t op_errno,
+			     struct iovec *vec,
+			     int32_t count,
+			     struct iatt *stbuf,
+			     struct iobref *iobref,
+			     dict_t *xdata)
+{
+	return rmw_partial_block(frame,
+				 cookie,
+				 this,
+				 op_ret,
+				 op_errno,
+				 vec,
+				 count,
+				 stbuf,
+				 iobref,
+				 atom_by_types(HOLE_ATOM, HEAD_ATOM));
+}
+
+static int32_t rmw_hole_tail(call_frame_t *frame,
+			     void *cookie,
+			     xlator_t *this,
+			     int32_t op_ret,
+			     int32_t op_errno,
+			     struct iovec *vec,
+			     int32_t count,
+			     struct iatt *stbuf,
+			     struct iobref *iobref,
+			     dict_t *xdata)
+{
+	return rmw_partial_block(frame,
+				 cookie,
+				 this,
+				 op_ret,
+				 op_errno,
+				 vec,
+				 count,
+				 stbuf,
+				 iobref,
+				 atom_by_types(HOLE_ATOM, TAIL_ATOM));
+}
+
+/*
+ * atom->count_to_uptodate()
+ */
+static uint32_t count_to_uptodate_head(struct avec_config *conf,
+				       struct object_cipher_info *object)
+{
+	if (conf->acount == 1 && conf->off_in_tail)
+		return get_atom_size(object);
+	else
+		/* there is no need to read the whole head block */
+		return conf->off_in_head;
+}
+
+static uint32_t count_to_uptodate_tail(struct avec_config *conf,
+				       struct object_cipher_info *object)
+{
+	/* we need to read the whole tail block */
+	return get_atom_size(object);
+}
+
+static uint32_t count_to_uptodate_data_head(call_frame_t *frame,
+					    struct object_cipher_info *object)
+{
+	return count_to_uptodate_head(get_data_conf(frame), object);
+}
+
+static uint32_t count_to_uptodate_data_tail(call_frame_t *frame,
+					    struct object_cipher_info *object)
+{
+	return count_to_uptodate_tail(get_data_conf(frame), object);
+}
+
+static uint32_t count_to_uptodate_hole_head(call_frame_t *frame,
+					    struct object_cipher_info *object)
+{
+	return count_to_uptodate_head(get_hole_conf(frame), object);
+}
+
+static uint32_t count_to_uptodate_hole_tail(call_frame_t *frame,
+					    struct object_cipher_info *object)
+{
+	return count_to_uptodate_tail(get_hole_conf(frame), object);
+}
+
+/* atom->get_config() */
+
+static struct avec_config *get_config_data(call_frame_t *frame)
+{
+	return &((crypt_local_t *)frame->local)->data_conf;
+}
+
+static struct avec_config *get_config_hole(call_frame_t *frame)
+{
+	return &((crypt_local_t *)frame->local)->hole_conf;
+}
+
+/*
+ * atom->get_iovec()
+ */
+static struct iovec *get_iovec_hole_head(call_frame_t *frame,
+					 uint32_t count)
+{
+	struct avec_config *conf = get_hole_conf(frame);
+
+	return conf->avec;
+}
+
+static struct iovec *get_iovec_hole_full(call_frame_t *frame,
+					 uint32_t count)
+{
+	struct avec_config *conf = get_hole_conf(frame);
+
+	return conf->avec + (conf->off_in_head ? 1 : 0);
+}
+
+static inline struct iovec *get_iovec_hole_tail(call_frame_t *frame,
+						uint32_t count)
+{
+	struct avec_config *conf = get_hole_conf(frame);
+
+	return conf->avec + (conf->blocks_in_pool - 1);
+}
+
+static inline struct iovec *get_iovec_data_head(call_frame_t *frame,
+						uint32_t count)
+{
+	struct avec_config *conf = get_data_conf(frame);
+
+	return conf->avec;
+}
+
+static inline struct iovec *get_iovec_data_full(call_frame_t *frame,
+						uint32_t count)
+{
+	struct avec_config *conf = get_data_conf(frame);
+
+	return conf->avec + (conf->off_in_head ? 1 : 0) + count;
+}
+
+static inline struct iovec *get_iovec_data_tail(call_frame_t *frame,
+						uint32_t count)
+{
+	struct avec_config *conf = get_data_conf(frame);
+
+	return conf->avec +
+		(conf->off_in_head ? 1 : 0) +
+		conf->nr_full_blocks;
+}
+
+static struct rmw_atom atoms[LAST_DATA_TYPE][LAST_LOCALITY_TYPE] = {
+	[DATA_ATOM][HEAD_ATOM] =
+	{ .locality = HEAD_ATOM,
+	  .rmw = rmw_data_head,
+	  .offset_at = offset_at_data_head,
+	  .offset_in = offset_in_data_head,
+	  .get_iovec = get_iovec_data_head,
+	  .io_size_nopad = io_size_nopad_data_head,
+	  .count_to_uptodate = count_to_uptodate_data_head,
+	  .get_config = get_config_data
+	},
+	[DATA_ATOM][TAIL_ATOM] =
+	{ .locality = TAIL_ATOM,
+	  .rmw = rmw_data_tail,
+	  .offset_at = offset_at_data_tail,
+	  .offset_in = offset_in_tail,
+	  .get_iovec = get_iovec_data_tail,
+	  .io_size_nopad = io_size_nopad_data_tail,
+	  .count_to_uptodate = count_to_uptodate_data_tail,
+	  .get_config = get_config_data
+	},
+	[DATA_ATOM][FULL_ATOM] =
+	{ .locality = FULL_ATOM,
+	  .offset_at = offset_at_data_full,
+	  .offset_in = offset_in_data_full,
+	  .get_iovec = get_iovec_data_full,
+	  .io_size_nopad = io_size_nopad_data_full,
+	  .get_config = get_config_data
+	},
+	[HOLE_ATOM][HEAD_ATOM] =
+	{ .locality = HEAD_ATOM,
+	  .rmw = rmw_hole_head,
+	  .offset_at = offset_at_hole_head,
+	  .offset_in = offset_in_hole_head,
+	  .get_iovec = get_iovec_hole_head,
+	  .io_size_nopad = io_size_nopad_hole_head,
+	  .count_to_uptodate = count_to_uptodate_hole_head,
+	  .get_config = get_config_hole
+	},
+	[HOLE_ATOM][TAIL_ATOM] =
+	{ .locality = TAIL_ATOM,
+	  .rmw = rmw_hole_tail,
+	  .offset_at = offset_at_hole_tail,
+	  .offset_in = offset_in_tail,
+	  .get_iovec = get_iovec_hole_tail,
+	  .io_size_nopad = io_size_nopad_hole_tail,
+	  .count_to_uptodate = count_to_uptodate_hole_tail,
+	  .get_config = get_config_hole
+	},
+	[HOLE_ATOM][FULL_ATOM] =
+	{ .locality = FULL_ATOM,
+	  .offset_at = offset_at_hole_full,
+	  .offset_in = offset_in_hole_full,
+	  .get_iovec = get_iovec_hole_full,
+	  .io_size_nopad = io_size_nopad_hole_full,
+	  .get_config = get_config_hole
+	}
+};
+
+struct rmw_atom *atom_by_types(atom_data_type data,
+			       atom_locality_type locality)
+{
+	return &atoms[data][locality];
+}
+
+/*
+  Local variables:
+  c-indentation-style: "K&R"
+  mode-name: "LC"
+  c-basic-offset: 8
+  tab-width: 8
+  fill-column: 80
+  scroll-step: 1
+  End:
+*/
diff --git a/xlators/encryption/crypt/src/crypt-common.h b/xlators/encryption/crypt/src/crypt-common.h
new file mode 100644
index 000000000..7c212ad5d
--- /dev/null
+++ b/xlators/encryption/crypt/src/crypt-common.h
@@ -0,0 +1,141 @@
+/*
+  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __CRYPT_COMMON_H__
+#define __CRYPT_COMMON_H__
+
+#define INVAL_SUBVERSION_NUMBER (0xff)
+#define CRYPT_INVAL_OP   (GF_FOP_NULL)
+
+#define CRYPTO_FORMAT_PREFIX  "trusted.glusterfs.crypt.att.cfmt"
+#define FSIZE_XATTR_PREFIX    "trusted.glusterfs.crypt.att.size"
+#define SUBREQ_PREFIX         "trusted.glusterfs.crypt.msg.sreq"
+#define FSIZE_MSG_PREFIX      "trusted.glusterfs.crypt.msg.size"
+#define DE_MSG_PREFIX         "trusted.glusterfs.crypt.msg.dent"
+#define REQUEST_ID_PREFIX     "trusted.glusterfs.crypt.msg.rqid"
+#define MSGFLAGS_PREFIX       "trusted.glusterfs.crypt.msg.xfgs"
+
+
+/* messages for crypt_open() */
+#define MSGFLAGS_REQUEST_MTD_RLOCK   1 /* take read lock and don't unlock */
+#define MSGFLAGS_REQUEST_MTD_WLOCK   2 /* take write lock and don't unlock */
+
+#define AES_BLOCK_BITS (4) /* AES_BLOCK_SIZE == 1 << AES_BLOCK_BITS */
+
+#define noop   do {; } while (0)
+#define cassert(cond) ({ switch (-1) { case (cond): case 0: break; } })
+#define __round_mask(x, y) ((__typeof__(x))((y)-1))
+#define round_up(x, y) ((((x)-1) | __round_mask(x, y))+1)
+
+/*
+ * Format of file's metadata
+ */
+struct crypt_format {
+	uint8_t loader_id;  /* version of metadata loader */
+	uint8_t versioned[0]; /* file's metadata of specific version */
+} __attribute__((packed));
+
+typedef enum {
+	AES_CIPHER_ALG,
+	LAST_CIPHER_ALG
+} cipher_alg_t;
+
+typedef enum {
+	XTS_CIPHER_MODE,
+	LAST_CIPHER_MODE
+} cipher_mode_t;
+
+typedef enum {
+	MTD_LOADER_V1,
+	LAST_MTD_LOADER
+} mtd_loader_id;
+
+static inline void msgflags_set_mtd_rlock(uint32_t *flags)
+{
+	*flags |= MSGFLAGS_REQUEST_MTD_RLOCK;
+}
+
+static inline void msgflags_set_mtd_wlock(uint32_t *flags)
+{
+	*flags |= MSGFLAGS_REQUEST_MTD_WLOCK;
+}
+
+static inline gf_boolean_t msgflags_check_mtd_rlock(uint32_t *flags)
+{
+	return *flags & MSGFLAGS_REQUEST_MTD_RLOCK;
+}
+
+static inline gf_boolean_t msgflags_check_mtd_wlock(uint32_t *flags)
+{
+	return *flags & MSGFLAGS_REQUEST_MTD_WLOCK;
+}
+
+static inline gf_boolean_t msgflags_check_mtd_lock(uint32_t *flags)
+{
+	return  msgflags_check_mtd_rlock(flags) ||
+		msgflags_check_mtd_wlock(flags);
+}
+
+/*
+ * returns number of logical blocks occupied
+ * (maybe partially) by @count bytes
+ * at offset @start.
+ */
+static inline off_t logical_blocks_occupied(uint64_t start, off_t count,
+					    int blkbits)
+{
+	return ((start + count - 1) >> blkbits) - (start >> blkbits) + 1;
+}
+
+/*
+ * are two bytes (represented by offsets @off1
+ * and @off2 respectively) in the same logical
+ * block.
+ */
+static inline int in_same_lblock(uint64_t off1, uint64_t off2,
+				 int blkbits)
+{
+	return off1 >> blkbits == off2 >> blkbits;
+}
+
+static inline void dump_cblock(xlator_t *this, unsigned char *buf)
+{
+	gf_log(this->name, GF_LOG_DEBUG,
+	       "dump cblock: %x %x %x %x %x %x %x %x %x %x %x %x %x %x %x %x",
+	       (buf)[0],
+	       (buf)[1],
+	       (buf)[2],
+	       (buf)[3],
+	       (buf)[4],
+	       (buf)[5],
+	       (buf)[6],
+	       (buf)[7],
+	       (buf)[8],
+	       (buf)[9],
+	       (buf)[10],
+	       (buf)[11],
+	       (buf)[12],
+	       (buf)[13],
+	       (buf)[14],
+	       (buf)[15]);
+}
+
+#endif /* __CRYPT_COMMON_H__ */
+
+/*
+  Local variables:
+  c-indentation-style: "K&R"
+  mode-name: "LC"
+  c-basic-offset: 8
+  tab-width: 8
+  fill-column: 80
+  scroll-step: 1
+  End:
+*/
diff --git a/xlators/encryption/crypt/src/crypt-mem-types.h b/xlators/encryption/crypt/src/crypt-mem-types.h
new file mode 100644
index 000000000..2eab921fc
--- /dev/null
+++ b/xlators/encryption/crypt/src/crypt-mem-types.h
@@ -0,0 +1,44 @@
+/*
+  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef __CRYPT_MEM_TYPES_H__
+#define __CRYPT_MEM_TYPES_H__
+
+#include "mem-types.h"
+
+enum gf_crypt_mem_types_ {
+	gf_crypt_mt_priv = gf_common_mt_end + 1,
+	gf_crypt_mt_inode,
+	gf_crypt_mt_data,
+	gf_crypt_mt_mtd,
+	gf_crypt_mt_loc,
+	gf_crypt_mt_iatt,
+	gf_crypt_mt_key,
+	gf_crypt_mt_iovec,
+	gf_crypt_mt_char,
+	gf_crypt_mt_end,
+};
+
+#endif /* __CRYPT_MEM_TYPES_H__ */
+
+/*
+  Local variables:
+  c-indentation-style: "K&R"
+  mode-name: "LC"
+  c-basic-offset: 8
+  tab-width: 8
+  fill-column: 80
+  scroll-step: 1
+  End:
+*/
+
+
+
diff --git a/xlators/encryption/crypt/src/crypt.c b/xlators/encryption/crypt/src/crypt.c
new file mode 100644
index 000000000..1abdad31d
--- /dev/null
+++ b/xlators/encryption/crypt/src/crypt.c
@@ -0,0 +1,4522 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+#include <ctype.h>
+#include <sys/uio.h>
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "xlator.h"
+#include "logging.h"
+#include "defaults.h"
+
+#include "crypt-common.h"
+#include "crypt.h"
+
+static void init_inode_info_head(struct crypt_inode_info *info, fd_t *fd);
+static int32_t init_inode_info_tail(struct crypt_inode_info *info,
+				    struct master_cipher_info *master);
+static int32_t prepare_for_submit_hole(call_frame_t *frame, xlator_t *this,
+				       uint64_t from, off_t size);
+static int32_t load_file_size(call_frame_t *frame, void *cookie,
+			      xlator_t *this, int32_t op_ret, int32_t op_errno,
+			      dict_t *dict, dict_t *xdata);
+static void do_ordered_submit(call_frame_t *frame, xlator_t *this,
+			      atom_data_type dtype);
+static void do_parallel_submit(call_frame_t *frame, xlator_t *this,
+			       atom_data_type dtype);
+static void put_one_call_open(call_frame_t *frame);
+static void put_one_call_readv(call_frame_t *frame, xlator_t *this);
+static void put_one_call_writev(call_frame_t *frame, xlator_t *this);
+static void put_one_call_ftruncate(call_frame_t *frame, xlator_t *this);
+static void free_avec(struct iovec *avec, char **pool, int blocks_in_pool);
+static void free_avec_data(crypt_local_t *local);
+static void free_avec_hole(crypt_local_t *local);
+
+static crypt_local_t *crypt_alloc_local(call_frame_t *frame, xlator_t *this,
+					glusterfs_fop_t fop)
+{
+	crypt_local_t *local = NULL;
+
+	local = mem_get0(this->local_pool);
+	if (!local) {
+		gf_log(this->name, GF_LOG_ERROR, "out of memory");
+		return NULL;
+	}
+	local->fop = fop;
+	LOCK_INIT(&local->hole_lock);
+	LOCK_INIT(&local->call_lock);
+	LOCK_INIT(&local->rw_count_lock);
+
+	frame->local = local;
+	return local;
+}
+
+struct crypt_inode_info *get_crypt_inode_info(inode_t *inode, xlator_t *this)
+{
+	int ret;
+	uint64_t value = 0;
+	struct crypt_inode_info *info;
+
+	ret = inode_ctx_get(inode, this, &value);
+	if (ret == -1) {
+		gf_log (this->name, GF_LOG_WARNING,
+			"Can not get inode info");
+		return NULL;
+	}
+	info = (struct crypt_inode_info *)(long)value;
+	if (info == NULL) {
+		gf_log (this->name, GF_LOG_WARNING,
+			"Can not obtain inode info");
+		return NULL;
+	}
+	return info;
+}
+
+static struct crypt_inode_info *local_get_inode_info(crypt_local_t *local,
+						      xlator_t *this)
+{
+	if (local->info)
+		return local->info;
+	local->info = get_crypt_inode_info(local->fd->inode, this);
+	return local->info;
+}
+
+static struct crypt_inode_info *alloc_inode_info(crypt_local_t *local,
+						 loc_t *loc)
+{
+	struct crypt_inode_info *info;
+
+	info = GF_CALLOC(1, sizeof(*info), gf_crypt_mt_inode);
+	if (!info) {
+		local->op_ret = -1;
+		local->op_errno = ENOMEM;
+		gf_log ("crypt", GF_LOG_WARNING,
+			"Can not allocate inode info");
+		return NULL;
+	}
+	memset(info, 0, sizeof(*info));
+#if DEBUG_CRYPT
+	info->loc = GF_CALLOC(1, sizeof(*loc), gf_crypt_mt_loc);
+	if (!info->loc) {
+		gf_log("crypt", GF_LOG_WARNING, "Can not allocate loc");
+		GF_FREE(info);
+		return NULL;
+	}
+	if (loc_copy(info->loc, loc)){
+		GF_FREE(info->loc);
+		GF_FREE(info);
+		return NULL;
+	}
+#endif /* DEBUG_CRYPT */
+
+	local->info = info;
+	return info;
+}
+
+static void free_inode_info(struct crypt_inode_info *info)
+{
+#if DEBUG_CRYPT
+	loc_wipe(info->loc);
+	GF_FREE(info->loc);
+#endif
+	memset(info, 0, sizeof(*info));
+	GF_FREE(info);
+}
+
+int crypt_forget (xlator_t *this, inode_t *inode)
+{
+        uint64_t ctx_addr = 0;
+        if (!inode_ctx_del (inode, this, &ctx_addr))
+                free_inode_info((struct crypt_inode_info *)(long)ctx_addr);
+        return 0;
+}
+
+#if DEBUG_CRYPT
+static void check_read(call_frame_t *frame, xlator_t *this, int32_t read,
+		       struct iovec *vec, int32_t count, struct iatt *stbuf)
+{
+	crypt_local_t *local = frame->local;
+	struct object_cipher_info *object = get_object_cinfo(local->info);
+	struct avec_config *conf = &local->data_conf;
+	uint32_t resid = stbuf->ia_size & (object_alg_blksize(object) - 1);
+
+	if (read <= 0)
+		return;
+	if (read != iovec_get_size(vec, count))
+		gf_log ("crypt", GF_LOG_DEBUG,
+			"op_ret differs from amount of read bytes");
+
+	if (object_alg_should_pad(object) && (read & (object_alg_blksize(object) - 1)))
+		gf_log ("crypt", GF_LOG_DEBUG,
+			"bad amount of read bytes (!= 0 mod(cblock size))");
+
+	if (conf->aligned_offset + read >
+	    stbuf->ia_size + (resid ? object_alg_blksize(object) - resid : 0))
+		gf_log ("crypt", GF_LOG_DEBUG,
+			"bad amount of read bytes (too large))");
+
+}
+
+#define PT_BYTES_TO_DUMP (32)
+static void dump_plain_text(crypt_local_t *local, struct iovec *avec)
+{
+	int32_t to_dump;
+	char str[PT_BYTES_TO_DUMP + 1];
+
+	if (!avec)
+		return;
+	to_dump = avec->iov_len;
+	if (to_dump > PT_BYTES_TO_DUMP)
+		to_dump = PT_BYTES_TO_DUMP;
+	memcpy(str, avec->iov_base, to_dump);
+	memset(str + to_dump, '0', 1);
+	gf_log("crypt", GF_LOG_DEBUG, "Read file: %s", str);
+}
+
+static int32_t data_conf_invariant(struct avec_config *conf)
+{
+	return conf->acount ==
+		!!has_head_block(conf) +
+		!!has_tail_block(conf)+
+		conf->nr_full_blocks;
+}
+
+static int32_t hole_conf_invariant(struct avec_config *conf)
+{
+	return conf->blocks_in_pool ==
+		!!has_head_block(conf) +
+		!!has_tail_block(conf)+
+		!!has_full_blocks(conf);
+}
+
+static void crypt_check_conf(struct avec_config *conf)
+{
+	int32_t ret = 0;
+	const char *msg;
+
+	switch (conf->type) {
+	case DATA_ATOM:
+		msg = "data";
+		ret = data_conf_invariant(conf);
+		break;
+	case HOLE_ATOM:
+		msg = "hole";
+		ret = hole_conf_invariant(conf);
+		break;
+	default:
+		msg = "unknown";
+	}
+	if (!ret)
+		gf_log("crypt", GF_LOG_DEBUG, "bad %s conf", msg);
+}
+
+static void check_buf(call_frame_t *frame, xlator_t *this, struct iatt *buf)
+{
+	crypt_local_t *local = frame->local;
+	struct object_cipher_info *object = &local->info->cinfo;
+	uint64_t local_file_size;
+
+	switch(local->fop) {
+	case GF_FOP_FTRUNCATE:
+		return;
+	case GF_FOP_WRITE:
+		local_file_size = local->new_file_size;
+		break;
+	case GF_FOP_READ:
+		if (parent_is_crypt_xlator(frame, this))
+			return;
+		local_file_size = local->cur_file_size;
+		break;
+	default:
+		gf_log("crypt", GF_LOG_DEBUG, "bad file operation");
+		return;
+	}
+	if (buf->ia_size != round_up(local_file_size,
+					 object_alg_blksize(object)))
+		gf_log("crypt", GF_LOG_DEBUG,
+		       "bad ia_size in buf (%llu), should be %llu",
+		       (unsigned long long)buf->ia_size,
+		       (unsigned long long)round_up(local_file_size,
+						    object_alg_blksize(object)));
+}
+
+#else
+#define check_read(frame, this, op_ret, vec, count, stbuf) noop
+#define dump_plain_text(local, avec) noop
+#define crypt_check_conf(conf) noop
+#define check_buf(frame, this, buf) noop
+#endif /* DEBUG_CRYPT */
+
+/*
+ * Pre-conditions:
+ * @vec represents a ciphertext of expanded size and
+ * aligned offset.
+ *
+ * Compound a temporal vector @avec with block-aligned
+ * components, decrypt and fix it up to represent a chunk
+ * of data corresponding to the original size and offset.
+ * Pass the result to the next translator.
+ */
+int32_t crypt_readv_cbk(call_frame_t *frame,
+			void *cookie,
+			xlator_t *this,
+			int32_t op_ret,
+			int32_t op_errno,
+			struct iovec *vec,
+			int32_t count,
+			struct iatt *stbuf,
+			struct iobref *iobref,
+			dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	struct avec_config *conf = &local->data_conf;
+	struct object_cipher_info *object = &local->info->cinfo;
+
+	struct iovec *avec;
+	uint32_t i;
+	uint32_t to_vec;
+	uint32_t to_user;
+
+	check_buf(frame, this, stbuf);
+	check_read(frame, this, op_ret, vec, count, stbuf);
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+	local->iobref = iobref_ref(iobref);
+
+	local->buf = *stbuf;
+	local->buf.ia_size = local->cur_file_size;
+
+	if (op_ret <= 0 || count == 0 || vec[0].iov_len == 0)
+		goto put_one_call;
+
+	if (conf->orig_offset >= local->cur_file_size) {
+		local->op_ret = 0;
+		goto put_one_call;
+	}
+	/* 
+	 * correct config params with real file size
+	 * and actual amount of bytes read
+	 */
+	set_config_offsets(frame, this,
+			   conf->orig_offset, op_ret, DATA_ATOM, 0);
+
+	if (conf->orig_offset + conf->orig_size > local->cur_file_size)
+		conf->orig_size = local->cur_file_size - conf->orig_offset;
+	/*
+	 * calculate amount of data to be returned
+	 * to user.
+	 */
+	to_user = op_ret;
+	if (conf->aligned_offset + to_user <= conf->orig_offset) {
+		gf_log(this->name, GF_LOG_WARNING, "Incomplete read");
+		local->op_ret = -1;
+		local->op_errno = EIO;
+		goto put_one_call;
+	}
+	to_user -= (conf->aligned_offset - conf->orig_offset);
+
+	if (to_user > conf->orig_size)
+		to_user = conf->orig_size;
+	local->rw_count = to_user;
+
+	op_errno = set_config_avec_data(this, local,
+					conf, object, vec, count);
+	if (op_errno) {
+		local->op_ret = -1;
+		local->op_errno = op_errno;
+		goto put_one_call;
+	}
+	avec = conf->avec;
+#if DEBUG_CRYPT
+	if (conf->off_in_tail != 0 &&
+	    conf->off_in_tail < object_alg_blksize(object) &&
+	    object_alg_should_pad(object))
+		gf_log(this->name, GF_LOG_DEBUG, "Bad offset in tail %d",
+		       conf->off_in_tail);
+	if (iovec_get_size(vec, count) != 0 &&
+	    in_same_lblock(conf->orig_offset + iovec_get_size(vec, count) - 1,
+			   local->cur_file_size - 1,
+			   object_alg_blkbits(object))) {
+		gf_log(this->name, GF_LOG_DEBUG, "Compound last cblock");
+		dump_cblock(this,
+			    (unsigned char *)(avec[conf->acount - 1].iov_base) +
+			    avec[conf->acount - 1].iov_len - object_alg_blksize(object));
+		dump_cblock(this,
+			    (unsigned char *)(vec[count - 1].iov_base) +
+			    vec[count - 1].iov_len - object_alg_blksize(object));
+	}
+#endif
+	decrypt_aligned_iov(object, avec,
+			    conf->acount, conf->aligned_offset);
+	/*
+	 * pass proper plain data to user
+	 */
+	avec[0].iov_base += (conf->aligned_offset - conf->orig_offset);
+	avec[0].iov_len  -= (conf->aligned_offset - conf->orig_offset);
+
+	to_vec = to_user;
+	for (i = 0; i < conf->acount; i++) {
+		if (avec[i].iov_len > to_vec)
+			avec[i].iov_len = to_vec;
+		to_vec -= avec[i].iov_len;
+	}
+ put_one_call:
+	put_one_call_readv(frame, this);
+	return 0;
+}
+
+static int32_t do_readv(call_frame_t *frame,
+			void *cookie,
+			xlator_t *this,
+			int32_t op_ret,
+			int32_t op_errno,
+			dict_t *dict,
+			dict_t *xdata)
+{
+	data_t *data;
+	crypt_local_t *local = frame->local;
+
+	if (op_ret < 0)
+		goto error;	
+	/*
+	 * extract regular file size
+	 */
+	data = dict_get(dict, FSIZE_XATTR_PREFIX);
+	if (!data) {
+		gf_log("crypt", GF_LOG_WARNING, "Regular file size not found");
+		op_errno = EIO;
+		goto error;
+	}
+	local->cur_file_size = data_to_uint64(data);
+	
+	get_one_call(frame);
+	STACK_WIND(frame,
+		   crypt_readv_cbk,
+		   FIRST_CHILD (this),
+		   FIRST_CHILD (this)->fops->readv,
+		   local->fd,
+		   /*
+		    * FIXME: read amount can be reduced
+		    */
+		   local->data_conf.expanded_size,
+		   local->data_conf.aligned_offset,
+		   local->flags,
+		   local->xdata);
+	return 0;
+ error:
+	local->op_ret = -1;
+	local->op_errno = op_errno;
+
+	get_one_call(frame);
+	put_one_call_readv(frame, this);
+	return 0;
+}
+
+static int32_t crypt_readv_finodelk_cbk(call_frame_t *frame,
+					void *cookie,
+					xlator_t *this,
+					int32_t op_ret,
+					int32_t op_errno,
+					dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	if (op_ret < 0)
+		goto error;
+	/*
+	 * An access has been granted,
+	 * retrieve file size
+	 */
+	STACK_WIND(frame,
+		   do_readv,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fgetxattr,
+		   local->fd,
+		   FSIZE_XATTR_PREFIX,
+		   NULL);
+	return  0;
+ error:
+	fd_unref(local->fd);
+	if (local->xdata)
+		dict_unref(local->xdata);
+	STACK_UNWIND_STRICT(readv,
+			    frame,
+			    -1,
+			    op_errno,
+			    NULL,
+			    0,
+			    NULL,
+			    NULL,
+			    NULL);
+	return 0;
+}
+
+static int32_t readv_trivial_completion(call_frame_t *frame,
+					void *cookie,
+					xlator_t *this,
+					int32_t op_ret,
+					int32_t op_errno,
+					struct iatt *buf,
+					dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (op_ret < 0) {
+		gf_log(this->name, GF_LOG_WARNING,
+		       "stat failed (%d)", op_errno);
+		goto error;
+	}
+	local->buf = *buf;
+	STACK_WIND(frame,
+		   load_file_size,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->getxattr,
+		   local->loc,
+		   FSIZE_XATTR_PREFIX,
+		   NULL);
+	return 0;	
+ error:
+	STACK_UNWIND_STRICT(readv, frame, op_ret, op_errno,
+			    NULL, 0, NULL, NULL, NULL);
+	return 0;
+}
+
+int32_t crypt_readv(call_frame_t *frame,
+		    xlator_t *this,
+		    fd_t *fd,
+		    size_t size,
+		    off_t offset,
+		    uint32_t flags, dict_t *xdata)
+{
+	int32_t ret;
+	crypt_local_t *local;
+	struct crypt_inode_info *info;
+	struct gf_flock lock = {0, };
+
+#if DEBUG_CRYPT
+	gf_log("crypt", GF_LOG_DEBUG, "reading %d bytes from offset %llu",
+	       (int)size, (long long)offset);
+	if (parent_is_crypt_xlator(frame, this))
+		gf_log("crypt", GF_LOG_DEBUG, "parent is crypt");
+#endif	
+	local = crypt_alloc_local(frame, this, GF_FOP_READ);
+	if (!local) {
+		ret = ENOMEM;
+		goto error;
+	}
+	if (size == 0)
+		goto trivial;
+
+	local->fd = fd_ref(fd);
+	local->flags = flags;
+
+	info = local_get_inode_info(local, this);
+	if (info == NULL) {
+		ret = EINVAL;
+		fd_unref(fd);
+		goto error;
+	}
+	if (!object_alg_atomic(&info->cinfo)) {
+		ret = EINVAL;
+		fd_unref(fd);
+		goto error;
+	}
+	set_config_offsets(frame, this, offset, size,
+			   DATA_ATOM, 0);
+	if (parent_is_crypt_xlator(frame, this)) {
+		data_t *data;
+		/*
+		 * We are called by crypt_writev (or cypt_ftruncate)
+		 * to perform the "read" component of the read-modify-write
+		 * (or read-prune-write) sequence for some atom;
+		 *
+		 * don't ask for access:
+		 * it has already been acquired
+		 *
+		 * Retrieve current file size
+		 */
+		if (!xdata) {
+			gf_log("crypt", GF_LOG_WARNING,
+			       "Regular file size hasn't been passed");
+			ret = EIO;
+			goto error;
+		}
+		data = dict_get(xdata, FSIZE_XATTR_PREFIX);
+		if (!data) {
+			gf_log("crypt", GF_LOG_WARNING,
+			       "Regular file size not found");
+			ret = EIO;
+			goto error;
+		}
+		local->old_file_size =
+			local->cur_file_size = data_to_uint64(data);
+
+		get_one_call(frame);
+		STACK_WIND(frame,
+			   crypt_readv_cbk,
+			   FIRST_CHILD(this),
+			   FIRST_CHILD(this)->fops->readv,
+			   local->fd,
+			   /*
+			    * FIXME: read amount can be reduced
+			    */
+			   local->data_conf.expanded_size,
+			   local->data_conf.aligned_offset,
+			   flags,
+			   NULL);
+		return 0;
+	}
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+
+	lock.l_len    = 0;
+        lock.l_start  = 0;
+        lock.l_type   = F_RDLCK;
+        lock.l_whence = SEEK_SET;
+
+	STACK_WIND(frame,
+		   crypt_readv_finodelk_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->finodelk,
+		   this->name,
+		   fd,
+		   F_SETLKW,
+		   &lock,
+		   NULL);
+	return 0;
+ trivial:
+	STACK_WIND(frame,
+		   readv_trivial_completion,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fstat,
+		   fd,
+		   NULL);
+	return 0;
+ error:
+	STACK_UNWIND_STRICT(readv,
+			    frame,
+			    -1,
+			    ret,
+			    NULL,
+			    0,
+			    NULL,
+			    NULL,
+			    NULL);
+	return 0;
+}
+
+void set_local_io_params_writev(call_frame_t *frame,
+				struct object_cipher_info *object,
+				struct rmw_atom *atom,
+				off_t io_offset,
+				uint32_t io_size)
+{
+	crypt_local_t *local = frame->local;
+
+	local->io_offset = io_offset;
+	local->io_size = io_size;
+
+	local->io_offset_nopad =
+		atom->offset_at(frame, object) + atom->offset_in(frame, object);
+
+	gf_log("crypt", GF_LOG_DEBUG,
+	       "set nopad offset to %llu",
+	       (unsigned long long)local->io_offset_nopad);
+
+	local->io_size_nopad = atom->io_size_nopad(frame, object);
+
+	gf_log("crypt", GF_LOG_DEBUG,
+	       "set nopad size to %llu",
+	       (unsigned long long)local->io_size_nopad);
+
+	local->update_disk_file_size = 0;
+	/*
+	 * NOTE: eof_padding_size is 0 for all full atoms;
+	 * For head and tail atoms it will be set up at rmw_partial block()
+	 */
+	local->new_file_size = local->cur_file_size;
+
+	if (local->io_offset_nopad + local->io_size_nopad > local->cur_file_size) {
+
+		local->new_file_size = local->io_offset_nopad + local->io_size_nopad;
+
+		gf_log("crypt", GF_LOG_DEBUG,
+		       "set new file size to %llu",
+		       (unsigned long long)local->new_file_size);
+
+		local->update_disk_file_size = 1;
+	}
+}
+
+void set_local_io_params_ftruncate(call_frame_t *frame,
+				   struct object_cipher_info *object)
+{
+	uint32_t resid;
+	crypt_local_t *local = frame->local;
+	struct avec_config *conf = &local->data_conf;
+
+	resid = conf->orig_offset & (object_alg_blksize(object) - 1);
+	if (resid) {
+		local->eof_padding_size =
+			object_alg_blksize(object) - resid;
+		local->new_file_size = conf->aligned_offset;
+		local->update_disk_file_size = 0;
+		/*
+		 * file size will be updated
+		 * in the ->writev() stack,
+		 * when submitting file tail
+		 */
+	}
+	else {	
+		local->eof_padding_size = 0;
+		local->new_file_size = conf->orig_offset;
+		local->update_disk_file_size = 1;
+		/*
+		 * file size will be updated
+		 * in this ->ftruncate stack
+		 */
+	}
+}
+
+static inline void submit_head(call_frame_t *frame, xlator_t *this)
+{
+	crypt_local_t *local = frame->local;
+	submit_partial(frame, this, local->fd, HEAD_ATOM);
+}
+
+static inline void submit_tail(call_frame_t *frame, xlator_t *this)
+{
+	crypt_local_t *local = frame->local;
+	submit_partial(frame, this, local->fd, TAIL_ATOM);
+}
+
+static void submit_hole(call_frame_t *frame, xlator_t *this)
+{
+	/*
+	 * hole conversion always means
+	 * appended write and goes in ordered fashion
+	 */
+	do_ordered_submit(frame, this, HOLE_ATOM);
+}
+
+static void submit_data(call_frame_t *frame, xlator_t *this)
+{
+	if (is_ordered_mode(frame)) {
+		do_ordered_submit(frame, this, DATA_ATOM);
+		return;
+	}
+	gf_log("crypt", GF_LOG_WARNING, "Bad submit mode");
+	get_nr_calls(frame, nr_calls_data(frame));
+	do_parallel_submit(frame, this, DATA_ATOM);
+	return;
+}
+
+/*
+ * heplers called by writev_cbk, fruncate_cbk in ordered mode
+ */
+
+static inline int32_t should_submit_hole(crypt_local_t *local)
+{
+	struct avec_config *conf = &local->hole_conf;
+
+	return conf->avec != NULL;
+}
+
+static inline int32_t should_resume_submit_hole(crypt_local_t *local)
+{
+	struct avec_config *conf = &local->hole_conf;
+
+	if (local->fop == GF_FOP_WRITE && has_tail_block(conf))
+		/*
+		 * Don't submit a part of hole, which
+		 * fits into a data block:
+ 		 * this part of hole will be converted
+		 * as a gap filled by zeros in data head
+		 * block.
+		 */
+		return conf->cursor < conf->acount - 1;
+	else
+		return conf->cursor < conf->acount;
+}
+
+static inline int32_t should_resume_submit_data(call_frame_t *frame)
+{
+	crypt_local_t *local = frame->local;
+	struct avec_config *conf = &local->data_conf;
+
+	if (is_ordered_mode(frame))
+		return conf->cursor < conf->acount;
+	/*
+	 * parallel writes
+	 */
+	return 0;
+}
+
+static inline int32_t should_submit_data_after_hole(crypt_local_t *local)
+{
+	return local->data_conf.avec != NULL;
+}
+
+static void update_local_file_params(call_frame_t *frame,
+				     xlator_t *this,
+				     struct iatt *prebuf,
+				     struct iatt *postbuf)
+{
+	crypt_local_t *local = frame->local;
+
+	check_buf(frame, this, postbuf);
+
+	local->prebuf = *prebuf;
+	local->postbuf = *postbuf;
+
+	local->prebuf.ia_size  = local->cur_file_size;
+	local->postbuf.ia_size = local->new_file_size;
+
+	local->cur_file_size = local->new_file_size;	
+}
+
+static int32_t end_writeback_writev(call_frame_t *frame,
+				    void *cookie,
+				    xlator_t *this,
+				    int32_t op_ret,
+				    int32_t op_errno,
+				    struct iatt *prebuf,
+				    struct iatt *postbuf,
+				    dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (op_ret <= 0) {
+		gf_log(this->name, GF_LOG_WARNING,
+		       "writev iteration failed");
+		goto put_one_call;
+	}
+	/*
+	 * op_ret includes paddings (atom's head, atom's tail and EOF)
+	 */
+	if (op_ret < local->io_size) {
+		gf_log(this->name, GF_LOG_WARNING,
+		       "Incomplete writev iteration");
+		goto put_one_call;
+	}
+	op_ret -= local->eof_padding_size;
+	local->op_ret = op_ret;
+
+	update_local_file_params(frame, this, prebuf, postbuf);
+
+	if (data_write_in_progress(local)) {
+
+		LOCK(&local->rw_count_lock);
+		local->rw_count += op_ret;
+		UNLOCK(&local->rw_count_lock);
+
+		if (should_resume_submit_data(frame))
+			submit_data(frame, this);
+	}
+	else {
+		/*
+		 * hole conversion is going on;
+		 * don't take into account written zeros
+		 */
+		if (should_resume_submit_hole(local))
+			submit_hole(frame, this);
+
+		else if (should_submit_data_after_hole(local))
+			submit_data(frame, this);
+	}
+ put_one_call:
+	put_one_call_writev(frame, this);
+	return 0;
+}
+
+#define crypt_writev_cbk end_writeback_writev
+
+#define HOLE_WRITE_CHUNK_BITS 12
+#define HOLE_WRITE_CHUNK_SIZE (1 << HOLE_WRITE_CHUNK_BITS)
+
+/*
+ * Convert hole of size @size at offset @off to
+ * zeros and prepare respective iovecs for submit.
+ * The hole lock should be held.
+ *
+ * Pre-conditions:
+ * @local->file_size is set and valid.
+ */
+int32_t prepare_for_submit_hole(call_frame_t *frame, xlator_t *this,
+				uint64_t off, off_t size)
+{
+	int32_t ret;
+	crypt_local_t *local = frame->local;
+	struct object_cipher_info *object = &local->info->cinfo;
+
+	set_config_offsets(frame, this, off, size, HOLE_ATOM, 1);
+
+	ret = set_config_avec_hole(this, local,
+				   &local->hole_conf, object, local->fop);
+	crypt_check_conf(&local->hole_conf);
+
+	return ret;
+}
+
+/*
+ * prepare for submit @count bytes at offset @from
+ */
+int32_t prepare_for_submit_data(call_frame_t *frame, xlator_t *this,
+				off_t from, int32_t size, struct iovec *vec,
+				int32_t vec_count, int32_t setup_gap)
+{
+	uint32_t ret;
+	crypt_local_t *local = frame->local;
+	struct object_cipher_info *object = &local->info->cinfo;
+
+	set_config_offsets(frame, this, from, size,
+			   DATA_ATOM, setup_gap);
+
+	ret = set_config_avec_data(this, local,
+				   &local->data_conf, object, vec, vec_count);
+	crypt_check_conf(&local->data_conf);
+
+	return ret;
+}
+
+static void free_avec(struct iovec *avec,
+		      char **pool, int blocks_in_pool)
+{
+	if (!avec)
+		return;
+	GF_FREE(pool);
+	GF_FREE(avec);
+}
+
+static void free_avec_data(crypt_local_t *local)
+{
+	return free_avec(local->data_conf.avec,
+			 local->data_conf.pool,
+			 local->data_conf.blocks_in_pool);
+}
+
+static void free_avec_hole(crypt_local_t *local)
+{
+	return free_avec(local->hole_conf.avec,
+			 local->hole_conf.pool,
+			 local->hole_conf.blocks_in_pool);
+}
+
+
+static void do_parallel_submit(call_frame_t *frame, xlator_t *this,
+			       atom_data_type dtype)
+{
+	crypt_local_t *local = frame->local;
+	struct avec_config *conf;
+
+	local->active_setup = dtype;
+	conf = conf_by_type(frame, dtype);
+
+	if (has_head_block(conf))
+		submit_head(frame, this);
+
+	if (has_full_blocks(conf))
+		submit_full(frame, this);
+
+	if (has_tail_block(conf))
+		submit_tail(frame, this);
+	return;
+}
+
+static void do_ordered_submit(call_frame_t *frame, xlator_t *this,
+			      atom_data_type dtype)
+{
+	crypt_local_t *local = frame->local;
+	struct avec_config *conf;
+
+	local->active_setup = dtype;
+	conf = conf_by_type(frame, dtype);
+
+	if (should_submit_head_block(conf)) {
+		get_one_call_nolock(frame);
+		submit_head(frame, this);
+	}
+	else if (should_submit_full_block(conf)) {
+		get_one_call_nolock(frame);
+		submit_full(frame, this);
+	}
+	else if (should_submit_tail_block(conf)) {
+		get_one_call_nolock(frame);
+		submit_tail(frame, this);
+	}
+	else
+		gf_log("crypt", GF_LOG_DEBUG,
+		       "nothing has been submitted in ordered mode");
+	return;
+}
+
+static int32_t do_writev(call_frame_t *frame,
+			 void *cookie,
+			 xlator_t *this,
+			 int32_t op_ret,
+			 int32_t op_errno,
+			 dict_t *dict,
+			 dict_t *xdata)
+{
+	data_t *data;
+	crypt_local_t *local = frame->local;
+	struct object_cipher_info *object = &local->info->cinfo;
+	/*
+	 * extract regular file size
+	 */
+	data = dict_get(dict, FSIZE_XATTR_PREFIX);
+	if (!data) {
+		gf_log("crypt", GF_LOG_WARNING, "Regular file size not found");
+		op_ret = -1;
+		op_errno = EIO;
+		goto error;
+	}
+	local->old_file_size = local->cur_file_size = data_to_uint64(data);
+
+	set_gap_at_end(frame, object, &local->data_conf, DATA_ATOM);
+
+	if (local->cur_file_size < local->data_conf.orig_offset) {
+		/*
+		 * Set up hole config
+		 */
+		op_errno = prepare_for_submit_hole(frame,
+			       this,
+			       local->cur_file_size,
+			       local->data_conf.orig_offset - local->cur_file_size);
+		if (op_errno) {
+			local->op_ret = -1;
+			local->op_errno = op_errno;
+			goto error;
+		}
+	}
+	if (should_submit_hole(local))
+		submit_hole(frame, this);
+	else 
+		submit_data(frame, this);
+	return 0;
+ error:
+	get_one_call_nolock(frame);
+	put_one_call_writev(frame, this);
+	return 0;
+}
+
+static int32_t crypt_writev_finodelk_cbk(call_frame_t *frame,
+					 void *cookie,
+					 xlator_t *this,
+					 int32_t op_ret,
+					 int32_t op_errno,
+					 dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (op_ret < 0)
+		goto error;
+	/*
+	 * An access has been granted,
+	 * retrieve file size first
+	 */
+	STACK_WIND(frame,
+		   do_writev,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fgetxattr,
+		   local->fd,
+		   FSIZE_XATTR_PREFIX,
+		   NULL);
+	return 0;
+ error:
+	get_one_call_nolock(frame);
+	put_one_call_writev(frame, this);
+	return 0;	
+}
+
+static int32_t writev_trivial_completion(call_frame_t *frame,
+					 void *cookie,
+					 xlator_t *this,
+					 int32_t op_ret,
+					 int32_t op_errno,
+					 struct iatt *buf,
+					 dict_t *dict)
+{
+	crypt_local_t *local = frame->local;
+	
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+	local->prebuf = *buf;
+	local->postbuf = *buf;
+
+	local->prebuf.ia_size = local->cur_file_size;
+	local->postbuf.ia_size = local->cur_file_size;
+
+	get_one_call(frame);
+	put_one_call_writev(frame, this);
+	return 0;
+}
+
+int crypt_writev(call_frame_t *frame,
+		 xlator_t *this,
+		 fd_t *fd,
+		 struct iovec *vec,
+		 int32_t count,
+		 off_t offset,
+		 uint32_t flags,
+		 struct iobref *iobref,
+		 dict_t *xdata)
+{
+	int32_t ret;
+	crypt_local_t *local;
+	struct crypt_inode_info *info;
+	struct gf_flock lock = {0, };
+#if DEBUG_CRYPT
+	gf_log ("crypt", GF_LOG_DEBUG, "writing %d bytes from offset %llu",
+		(int)iovec_get_size(vec, count), (long long)offset);
+#endif
+	local = crypt_alloc_local(frame, this, GF_FOP_WRITE);
+	if (!local) {
+		ret = ENOMEM;
+		goto error;
+	}
+	local->fd = fd_ref(fd);
+
+	if (iobref)
+		local->iobref = iobref_ref(iobref);
+	/*
+	 * to update real file size on the server
+	 */
+	local->xattr = dict_new();
+	if (!local->xattr) {
+		ret = ENOMEM;
+		goto error;
+	}
+	local->flags = flags;
+
+	info = local_get_inode_info(local, this);
+	if (info == NULL) {
+		ret = EINVAL;
+		goto error;
+	}
+	if (!object_alg_atomic(&info->cinfo)) {
+		ret = EINVAL;
+		goto error;
+	}
+	if (iovec_get_size(vec, count) == 0)
+		goto trivial;
+
+	ret = prepare_for_submit_data(frame, this, offset,
+				      iovec_get_size(vec, count),
+				      vec, count, 0 /* don't setup gup
+						       in tail: we don't
+						       know file size yet */);
+	if (ret)
+		goto error;
+
+	if (parent_is_crypt_xlator(frame, this)) {
+		data_t *data;
+		/*
+		 * we are called by shinking crypt_ftruncate(),
+		 * which doesn't perform hole conversion;
+		 *
+		 * don't ask for access:
+		 * it has already been acquired
+		 */
+
+		/*
+		 * extract file size
+		 */
+		if (!xdata) {
+			gf_log("crypt", GF_LOG_WARNING,
+			       "Regular file size hasn't been passed");
+			ret = EIO;
+			goto error;
+		}
+		data = dict_get(xdata, FSIZE_XATTR_PREFIX);
+		if (!data) {
+			gf_log("crypt", GF_LOG_WARNING,
+			       "Regular file size not found");
+			ret = EIO;
+			goto error;
+		}
+		local->old_file_size =
+			local->cur_file_size = data_to_uint64(data);
+
+		submit_data(frame, this);
+		return 0;
+	}
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+	/*
+	 * lock the file and retrieve its size
+	 */
+	lock.l_len    = 0;
+        lock.l_start  = 0;
+        lock.l_type   = F_WRLCK;
+        lock.l_whence = SEEK_SET;
+
+	STACK_WIND(frame,
+		   crypt_writev_finodelk_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->finodelk,
+		   this->name,
+		   fd,
+		   F_SETLKW,
+		   &lock,
+		   NULL);
+	return 0;
+ trivial:
+	STACK_WIND(frame,
+		   writev_trivial_completion,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fstat,
+		   fd,
+		   NULL);
+	return 0;
+ error:
+	if (local && local->fd)
+		fd_unref(fd);
+	if (local && local->iobref)
+		iobref_unref(iobref);
+	if (local && local->xdata)
+		dict_unref(xdata);
+	if (local && local->xattr)
+		dict_unref(local->xattr);
+	if (local && local->info)
+		free_inode_info(local->info);
+
+	STACK_UNWIND_STRICT(writev, frame, -1, ret, NULL, NULL, NULL);
+	return 0;
+}
+
+int32_t prepare_for_prune(call_frame_t *frame, xlator_t *this, uint64_t offset)
+{
+	set_config_offsets(frame, this,
+			   offset,
+			   0, /* count */
+			   DATA_ATOM,
+			   0 /* since we prune, there is no
+				gap in tail to uptodate */);
+	return 0;
+}
+
+/*
+ * Finish the read-prune-modify sequence
+ *
+ * Can be invoked as
+ * 1) ->ftruncate_cbk() for cblock-aligned, or trivial prune
+ * 2) ->writev_cbk() for non-cblock-aligned prune
+ */
+
+static int32_t prune_complete(call_frame_t *frame,
+			      void *cookie,
+			      xlator_t *this,
+			      int32_t op_ret,
+			      int32_t op_errno,
+			      struct iatt *prebuf,
+			      struct iatt *postbuf,
+			      dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	update_local_file_params(frame, this, prebuf, postbuf);
+
+	put_one_call_ftruncate(frame, this);
+	return 0;
+}
+
+/*
+ * This is called as ->ftruncate_cbk()
+ *
+ * Perform the "write" component of the
+ * read-prune-write sequence.
+ *
+ * submuit the rest of the file
+ */
+static int32_t prune_submit_file_tail(call_frame_t *frame,
+				      void *cookie,
+				      xlator_t *this,
+				      int32_t op_ret,
+				      int32_t op_errno,
+				      struct iatt *prebuf,
+				      struct iatt *postbuf,
+				      dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	struct avec_config *conf = &local->data_conf;
+	dict_t *dict;
+
+	if (op_ret < 0)
+		goto put_one_call;
+
+	if (local->xdata) {
+		dict_unref(local->xdata);
+		local->xdata = NULL;
+	}
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+
+	dict = dict_new();
+	if (!dict) {
+		op_errno = ENOMEM;
+		goto error;
+	}
+
+	update_local_file_params(frame, this, prebuf, postbuf);
+	local->new_file_size = conf->orig_offset;
+
+	/*
+	 * The rest of the file is a partial block and, hence,
+	 * should be written via RMW sequence, so the crypt xlator
+	 * does STACK_WIND to itself.
+	 *
+	 * Pass current file size to crypt_writev()
+	 */
+	op_errno = dict_set(dict,
+			    FSIZE_XATTR_PREFIX,
+			    data_from_uint64(local->cur_file_size));
+	if (op_errno) {
+		gf_log("crypt", GF_LOG_WARNING,
+		       "can not set key to update file size");
+		dict_unref(dict);
+		goto error;
+	}
+	gf_log("crypt", GF_LOG_DEBUG,
+	       "passing current file size (%llu) to crypt_writev",
+	       (unsigned long long)local->cur_file_size);
+	/*
+	 * Padding will be filled with
+	 * zeros by rmw_partial_block()
+	 */
+	STACK_WIND(frame,
+		   prune_complete,
+		   this,
+		   this->fops->writev, /* crypt_writev */
+		   local->fd,
+		   &local->vec,
+		   1,
+		   conf->aligned_offset, /* offset to write from */
+		   0,
+		   local->iobref,
+		   dict);
+
+	dict_unref(dict);
+	return 0;
+ error:
+	local->op_ret = -1;
+	local->op_errno = op_errno;
+ put_one_call:
+	put_one_call_ftruncate(frame, this);
+	return 0;
+}
+
+/*
+ * This is called as a callback of ->writev() invoked in behalf
+ * of ftruncate(): it can be
+ * 1) ordered writes issued by hole conversion in the case of
+ *    expanded truncate, or
+ * 2) an rmw partial data block issued by non-cblock-aligned
+ * prune.
+ */
+int32_t end_writeback_ftruncate(call_frame_t *frame,
+				void *cookie,
+				xlator_t *this,
+				int32_t op_ret,
+				int32_t op_errno,
+				struct iatt *prebuf,
+				struct iatt *postbuf,
+				dict_t *xdata)
+{
+	crypt_local_t  *local = frame->local;
+	/*
+	 * if nothing has been written,
+	 * then it must be an error
+	 */
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (op_ret < 0)
+		goto put_one_call;
+
+	update_local_file_params(frame, this, prebuf, postbuf);
+
+	if (data_write_in_progress(local))
+		/* case (2) */
+		goto put_one_call;
+	/* case (1) */
+	if (should_resume_submit_hole(local))
+		submit_hole(frame, this);
+	/*
+	 * case of hole, when we should't resume
+	 */
+ put_one_call:
+	put_one_call_ftruncate(frame, this);
+	return 0;
+}
+
+/*
+ * Perform prune and write components of the
+ * read-prune-write sequence.
+ *
+ * Called as ->readv_cbk()
+ *
+ * Pre-conditions:
+ * @vec contains the latest atom of the file
+ * (plain text)
+ */
+static int32_t prune_write(call_frame_t *frame,
+			   void *cookie,
+			   xlator_t *this,
+			   int32_t op_ret,
+			   int32_t op_errno,
+			   struct iovec *vec,
+			   int32_t count,
+			   struct iatt *stbuf,
+			   struct iobref *iobref,
+			   dict_t *xdata)
+{
+	int32_t i;
+	size_t to_copy;
+	size_t copied = 0;
+	crypt_local_t *local = frame->local;
+	struct avec_config *conf = &local->data_conf;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+	if (op_ret == -1)
+		goto put_one_call;
+
+	/*
+	 * At first, uptodate head block
+	 */
+	if (iovec_get_size(vec, count) < conf->off_in_head) {
+		gf_log(this->name, GF_LOG_WARNING,
+		       "Failed to uptodate head block for prune");
+		local->op_ret = -1;
+		local->op_errno = EIO;
+		goto put_one_call;
+	}
+	local->vec.iov_len = conf->off_in_head;
+	local->vec.iov_base = GF_CALLOC(1, local->vec.iov_len,
+					gf_crypt_mt_data);
+
+	if (local->vec.iov_base == NULL) {
+	        gf_log(this->name, GF_LOG_WARNING,
+                       "Failed to calloc head block for prune");
+		local->op_ret = -1;
+		local->op_errno = ENOMEM;
+	        goto put_one_call;
+	}
+	for (i = 0; i < count; i++) {
+		to_copy = vec[i].iov_len;
+		if (to_copy > local->vec.iov_len - copied)
+			to_copy = local->vec.iov_len - copied;
+
+		memcpy((char *)local->vec.iov_base + copied,
+		       vec[i].iov_base,
+		       to_copy);
+		copied += to_copy;
+		if (copied == local->vec.iov_len)
+			break;
+	}
+	/*
+	 * perform prune with aligned offset
+	 * (i.e. at this step we prune a bit
+	 * more then it is needed
+	 */
+	STACK_WIND(frame,
+		   prune_submit_file_tail,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->ftruncate,
+		   local->fd,
+		   conf->aligned_offset,
+		   local->xdata);
+	return 0;
+ put_one_call:
+	put_one_call_ftruncate(frame, this);
+	return 0;
+}
+
+/*
+ * Perform a read-prune-write sequence
+ */
+int32_t read_prune_write(call_frame_t *frame, xlator_t *this)
+{
+	int32_t ret = 0;
+	dict_t *dict = NULL;
+	crypt_local_t  *local = frame->local;
+	struct avec_config *conf = &local->data_conf;
+	struct object_cipher_info *object = &local->info->cinfo;
+
+	set_local_io_params_ftruncate(frame, object);
+	get_one_call_nolock(frame);
+
+	if ((conf->orig_offset & (object_alg_blksize(object) - 1)) == 0) {
+		/*
+		 * cblock-aligned prune:
+		 * we don't need read and write components,
+		 * just cut file body
+		 */
+		gf_log("crypt", GF_LOG_DEBUG,
+		       "prune without RMW (at offset %llu",
+		       (unsigned long long)conf->orig_offset);
+
+		STACK_WIND(frame,
+			   prune_complete,
+			   FIRST_CHILD(this),
+			   FIRST_CHILD(this)->fops->ftruncate,
+			   local->fd,
+			   conf->orig_offset,
+			   local->xdata);
+		return 0;
+	}
+	gf_log("crypt", GF_LOG_DEBUG,
+	       "prune with RMW (at offset %llu",
+	       (unsigned long long)conf->orig_offset);	
+	/*
+	 * We are about to perform the "read" component of the
+	 * read-prune-write sequence. It means that we need to
+	 * read encrypted data from disk and decrypt it.
+	 * So, the crypt translator does STACK_WIND to itself.
+	 * 
+	 * Pass current file size to crypt_readv()
+	 
+	 */
+	dict = dict_new();
+	if (!dict) {
+		gf_log("crypt", GF_LOG_WARNING, "Can not alloc dict");
+		ret = ENOMEM;
+		goto exit;
+	}
+	ret = dict_set(dict,
+		       FSIZE_XATTR_PREFIX,
+		       data_from_uint64(local->cur_file_size));
+	if (ret) {
+		gf_log("crypt", GF_LOG_WARNING, "Can not set dict");
+		goto exit;
+	}
+	STACK_WIND(frame,
+		   prune_write,
+		   this,
+		   this->fops->readv, /* crypt_readv */
+		   local->fd,
+		   get_atom_size(object), /* bytes to read */
+		   conf->aligned_offset, /* offset to read from */
+		   0,
+		   dict);
+ exit:
+	if (dict)
+		dict_unref(dict);
+	return ret;
+}
+
+/*
+ * File prune is more complicated than expand.
+ * First we need to read the latest atom to not lose info
+ * needed for proper update. Also we need to make sure that
+ * every component of read-prune-write sequence leaves data
+ * consistent
+ *
+ * Non-cblock aligned prune is performed as read-prune-write
+ * sequence:
+ *
+ * 1) read the latest atom;
+ * 2) perform cblock-aligned prune
+ * 3) issue a write request for the end-of-file
+ */
+int32_t prune_file(call_frame_t *frame, xlator_t *this, uint64_t offset)
+{
+	int32_t ret;
+
+	ret = prepare_for_prune(frame, this, offset);
+	if (ret)
+		return ret;
+	return read_prune_write(frame, this);
+}
+
+int32_t expand_file(call_frame_t *frame, xlator_t *this,
+		    uint64_t offset)
+{
+	int32_t ret;
+	crypt_local_t *local = frame->local;
+
+	ret = prepare_for_submit_hole(frame, this,
+				      local->old_file_size,
+				      offset - local->old_file_size);
+	if (ret)
+		return ret;
+	submit_hole(frame, this);
+	return 0;
+}
+
+static int32_t ftruncate_trivial_completion(call_frame_t *frame,
+					    void *cookie,
+					    xlator_t *this,
+					    int32_t op_ret,
+					    int32_t op_errno,
+					    struct iatt *buf,
+					    dict_t *dict)
+{
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+	local->prebuf = *buf;
+	local->postbuf = *buf;
+
+	local->prebuf.ia_size = local->cur_file_size;
+	local->postbuf.ia_size = local->cur_file_size;
+
+	get_one_call(frame);
+	put_one_call_ftruncate(frame, this);
+	return 0;
+}
+
+static int32_t do_ftruncate(call_frame_t *frame,
+			    void *cookie,
+			    xlator_t *this,
+			    int32_t op_ret,
+			    int32_t op_errno,
+			    dict_t *dict,
+			    dict_t *xdata)
+{
+	data_t *data;
+	crypt_local_t *local = frame->local;
+
+	if (op_ret)
+		goto error;	
+	/*
+	 * extract regular file size
+	 */
+	data = dict_get(dict, FSIZE_XATTR_PREFIX);
+	if (!data) {
+		gf_log("crypt", GF_LOG_WARNING, "Regular file size not found");
+		op_errno = EIO;
+		goto error;
+	}
+	local->old_file_size = local->cur_file_size = data_to_uint64(data);
+
+	if (local->data_conf.orig_offset == local->cur_file_size) {
+#if DEBUG_CRYPT
+		gf_log("crypt", GF_LOG_DEBUG,
+		       "trivial ftruncate (current file size %llu)",
+		       (unsigned long long)local->cur_file_size);
+#endif
+		goto trivial;
+	}
+	else if (local->data_conf.orig_offset < local->cur_file_size) {
+#if DEBUG_CRYPT
+		gf_log("crypt", GF_LOG_DEBUG, "prune from %llu to %llu",
+		       (unsigned long long)local->cur_file_size,
+		       (unsigned long long)local->data_conf.orig_offset);
+#endif
+		op_errno = prune_file(frame,
+				      this,
+				      local->data_conf.orig_offset);
+	}
+	else {
+#if DEBUG_CRYPT
+		gf_log("crypt", GF_LOG_DEBUG, "expand from %llu to %llu",
+		       (unsigned long long)local->cur_file_size,
+		       (unsigned long long)local->data_conf.orig_offset);
+#endif
+		op_errno = expand_file(frame,
+				       this,
+				       local->data_conf.orig_offset);
+	}
+	if (op_errno)
+		goto error;
+	return 0;
+ trivial:
+	STACK_WIND(frame,
+		   ftruncate_trivial_completion,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fstat,
+		   local->fd,
+		   NULL);
+	return 0;
+ error:
+	/*
+	 * finish with ftruncate
+	 */
+	local->op_ret = -1;
+	local->op_errno = op_errno;
+
+	get_one_call_nolock(frame);
+	put_one_call_ftruncate(frame, this);
+	return 0;
+}
+
+static int32_t crypt_ftruncate_finodelk_cbk(call_frame_t *frame,
+					    void *cookie,
+					    xlator_t *this,
+					    int32_t op_ret,
+					    int32_t op_errno,
+					    dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (op_ret < 0)
+		goto error;
+	/*
+	 * An access has been granted,
+	 * retrieve file size first
+	 */
+	STACK_WIND(frame,
+		   do_ftruncate,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fgetxattr,
+		   local->fd,
+		   FSIZE_XATTR_PREFIX,
+		   NULL);
+	return 0;
+ error:
+	get_one_call_nolock(frame);
+	put_one_call_ftruncate(frame, this);
+	return 0;
+}
+
+/*
+ * ftruncate is performed in 2 steps:
+ * . recieve file size;
+ * . expand or prune file.
+ */
+static int32_t crypt_ftruncate(call_frame_t *frame,
+			       xlator_t *this,
+			       fd_t *fd,
+			       off_t offset,
+			       dict_t *xdata)
+{
+	int32_t ret;
+	crypt_local_t *local;
+	struct crypt_inode_info *info;
+	struct gf_flock lock = {0, };
+
+	local = crypt_alloc_local(frame, this, GF_FOP_FTRUNCATE);
+	if (!local) {
+		ret = ENOMEM;
+		goto error;
+	}
+	local->xattr = dict_new();
+	if (!local->xattr) {
+		ret = ENOMEM;
+		goto error;
+	}
+	local->fd = fd_ref(fd);
+	info = local_get_inode_info(local, this);
+	if (info == NULL) {
+		ret = EINVAL;
+		goto error;
+	}
+	if (!object_alg_atomic(&info->cinfo)) {
+		ret = EINVAL;
+		goto error;
+	}
+	local->data_conf.orig_offset = offset;
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+
+	lock.l_len    = 0;
+        lock.l_start  = 0;
+        lock.l_type   = F_WRLCK;
+        lock.l_whence = SEEK_SET;
+
+	STACK_WIND(frame,
+		   crypt_ftruncate_finodelk_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->finodelk,
+		   this->name,
+		   fd,
+		   F_SETLKW,
+		   &lock,
+		   NULL);
+	return 0;
+ error:
+	if (local && local->fd)
+		fd_unref(fd);
+	if (local && local->xdata)
+		dict_unref(xdata);
+	if (local && local->xattr)
+		dict_unref(local->xattr);
+	if (local && local->info)
+		free_inode_info(local->info);
+
+	STACK_UNWIND_STRICT(ftruncate, frame, -1, ret, NULL, NULL, NULL);
+	return 0;
+}
+
+/* ->flush_cbk() */
+int32_t truncate_end(call_frame_t *frame,
+		     void *cookie,
+		     xlator_t *this,
+		     int32_t op_ret,
+		     int32_t op_errno,
+		     dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	STACK_UNWIND_STRICT(truncate,
+			    frame,
+			    op_ret,
+			    op_errno,
+			    &local->prebuf,
+			    &local->postbuf,
+			    local->xdata);
+	return 0;
+}
+
+/* ftruncate_cbk() */
+int32_t truncate_flush(call_frame_t *frame,
+		       void *cookie,
+		       xlator_t *this,
+		       int32_t op_ret,
+		       int32_t op_errno,
+		       struct iatt *prebuf,
+		       struct iatt *postbuf,
+		       dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	fd_t *fd = local->fd;
+	local->prebuf = *prebuf;
+	local->postbuf = *postbuf;
+
+	STACK_WIND(frame,
+		   truncate_end,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->flush,
+		   fd,
+		   NULL);
+	fd_unref(fd);
+	return 0;
+}
+
+/*
+ * is called as ->open_cbk()
+ */
+static int32_t truncate_begin(call_frame_t *frame,
+			      void *cookie,
+			      xlator_t *this,
+			      int32_t op_ret,
+			      int32_t op_errno,
+			      fd_t *fd,
+			      dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	if (op_ret < 0) {
+		fd_unref(fd);
+		STACK_UNWIND_STRICT(truncate,
+				    frame,
+				    op_ret,
+				    op_errno, NULL, NULL, NULL);
+		return 0;
+	}
+	/*
+	 * crypt_truncate() is implemented via crypt_ftruncate(),
+	 * so the crypt xlator does STACK_WIND to itself here
+	 */
+	STACK_WIND(frame,
+		   truncate_flush,
+		   this,
+		   this->fops->ftruncate, /* crypt_ftruncate */
+		   fd,
+		   local->offset,
+		   NULL);
+	return 0;
+}
+
+/*
+ * crypt_truncate() is implemented via crypt_ftruncate() as a
+ * sequence crypt_open() - crypt_ftruncate() - truncate_flush()
+ */
+int32_t crypt_truncate(call_frame_t *frame,
+		       xlator_t *this,
+		       loc_t *loc,
+		       off_t offset,
+		       dict_t *xdata)
+{
+	fd_t *fd;
+	crypt_local_t *local;
+
+#if DEBUG_CRYPT
+	gf_log(this->name, GF_LOG_DEBUG,
+	       "truncate file %s at offset %llu",
+	       loc->path, (unsigned long long)offset);
+#endif
+	local = crypt_alloc_local(frame, this, GF_FOP_TRUNCATE);
+	if (!local)
+		goto error;
+
+	fd = fd_create(loc->inode, frame->root->pid);
+	if (!fd) {
+		gf_log(this->name, GF_LOG_ERROR, "Can not create fd");
+		goto error;
+	}
+	local->fd = fd;
+	local->offset = offset;
+	local->xdata = xdata;
+	STACK_WIND(frame,
+		   truncate_begin,
+		   this,
+		   this->fops->open, /* crypt_open() */
+		   loc,
+		   O_RDWR,
+		   fd,
+		   NULL);
+	return 0;
+ error:
+	STACK_UNWIND_STRICT(truncate, frame, -1, EINVAL, NULL, NULL, NULL);
+	return 0;
+}
+
+end_writeback_handler_t dispatch_end_writeback(glusterfs_fop_t fop)
+{
+	switch (fop) {
+	case GF_FOP_WRITE:
+		return end_writeback_writev;
+	case GF_FOP_FTRUNCATE:
+		return end_writeback_ftruncate;
+	default:
+		gf_log("crypt", GF_LOG_WARNING, "Bad wb operation %d", fop);
+		return NULL;
+	}
+}
+
+/*
+ * true, if the caller needs metadata string
+ */
+static int32_t is_custom_mtd(dict_t *xdata)
+{
+	data_t *data;
+	uint32_t flags;
+
+	if (!xdata)
+		return 0;
+
+	data = dict_get(xdata, MSGFLAGS_PREFIX);
+	if (!data)
+		return 0;
+	if (data->len != sizeof(uint32_t)) {
+		gf_log("crypt", GF_LOG_WARNING,
+		       "Bad msgflags size (%d)", data->len);
+		return -1;
+	}
+	flags = *((uint32_t *)data->data);
+	return msgflags_check_mtd_lock(&flags);
+}
+
+static int32_t crypt_open_done(call_frame_t *frame,
+			       void *cookie,
+			       xlator_t *this,
+			       int32_t op_ret,
+			       int32_t op_errno, dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+	if (op_ret < 0)
+		gf_log(this->name, GF_LOG_WARNING, "mtd unlock failed (%d)",
+		       op_errno);
+	put_one_call_open(frame);
+	return 0;
+}
+
+static void crypt_open_tail(call_frame_t *frame, xlator_t *this)
+{
+	struct gf_flock  lock  = {0, };
+	crypt_local_t *local = frame->local;
+
+        lock.l_type   = F_UNLCK;
+        lock.l_whence = SEEK_SET;
+        lock.l_start  = 0;
+        lock.l_len    = 0;
+        lock.l_pid    = 0;
+
+	STACK_WIND(frame,
+		   crypt_open_done,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->finodelk,
+		   this->name,
+		   local->fd,
+		   F_SETLKW,
+		   &lock,
+		   NULL);
+}
+
+/*
+ * load private inode info at open time
+ * called as ->fgetxattr_cbk()
+ */
+static int load_mtd_open(call_frame_t *frame,
+			 void *cookie,
+			 xlator_t *this,
+			 int32_t op_ret,
+			 int32_t op_errno,
+			 dict_t *dict,
+			 dict_t *xdata)
+{
+	int32_t ret;
+	gf_boolean_t upload_info;
+	data_t *mtd;
+	uint64_t value = 0;
+	struct crypt_inode_info *info;
+	crypt_local_t *local = frame->local;
+	crypt_private_t *priv = this->private;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (local->fd->inode->ia_type == IA_IFLNK)
+		goto exit;
+	if (op_ret < 0)
+		goto exit;
+	/*
+	 * first, check for cached info
+	 */
+	ret = inode_ctx_get(local->fd->inode, this, &value);
+	if (ret != -1) {
+		info = (struct crypt_inode_info *)(long)value;
+		if (info == NULL) {
+			gf_log(this->name, GF_LOG_WARNING,
+			       "Inode info expected, but not found");
+			local->op_ret = -1;
+			local->op_errno = EIO;
+			goto exit;
+		}
+		/*
+		 * info has been found in the cache
+		 */
+		upload_info = _gf_false;
+	}
+	else {
+		/*
+		 * info hasn't been found in the cache.
+		 */
+		info = alloc_inode_info(local, local->loc);
+		if (!info) {
+			local->op_ret = -1;
+			local->op_errno = ENOMEM;
+			goto exit;
+		}
+		init_inode_info_head(info, local->fd);
+		upload_info = _gf_true;
+	}
+	/*
+	 * extract metadata
+	 */
+	mtd = dict_get(dict, CRYPTO_FORMAT_PREFIX);
+	if (!mtd) {
+		local->op_ret = -1;
+		local->op_errno = ENOENT;
+		gf_log (this->name, GF_LOG_WARNING,
+			"Format string wasn't found");
+		goto exit;
+	}
+	/*
+	 * authenticate metadata against the path
+	 */	
+	ret = open_format((unsigned char *)mtd->data,
+			  mtd->len,
+			  local->loc,
+			  info,
+			  get_master_cinfo(priv),
+			  local,
+			  upload_info);
+	if (ret) {
+		local->op_ret = -1;
+		local->op_errno = ret;
+		goto exit;
+	}
+	if (upload_info) {
+		ret = init_inode_info_tail(info, get_master_cinfo(priv));
+		if (ret) {
+			local->op_ret = -1;
+			local->op_errno = ret;
+			goto exit;	
+		}
+		ret = inode_ctx_put(local->fd->inode,
+				    this, (uint64_t)(long)info);
+		if (ret == -1) {
+			local->op_ret = -1;
+			local->op_errno = EIO;
+			goto exit;
+		}
+	}
+	if (local->custom_mtd) {
+		/*
+		 * pass the metadata string to the customer
+		 */
+		ret = dict_set_static_bin(local->xdata,
+					  CRYPTO_FORMAT_PREFIX,
+					  mtd->data,
+					  mtd->len);
+		if (ret) {
+			local->op_ret = -1;
+			local->op_errno = ret;
+			goto exit;
+		}
+	}
+ exit:
+	if (!local->custom_mtd)
+		crypt_open_tail(frame, this);
+	else
+		put_one_call_open(frame);
+	return 0;
+}
+
+static int32_t crypt_open_finodelk_cbk(call_frame_t *frame,
+				       void *cookie,
+				       xlator_t *this,
+				       int32_t op_ret,
+				       int32_t op_errno,
+				       dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (op_ret < 0) {
+		gf_log(this->name, GF_LOG_WARNING, "finodelk (LOCK) failed");
+		goto exit;
+	}
+	STACK_WIND(frame,
+		   load_mtd_open,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fgetxattr,
+		   local->fd,
+		   CRYPTO_FORMAT_PREFIX,
+		   NULL);
+	return 0;
+ exit:
+	put_one_call_open(frame);
+	return 0;
+}
+
+/*
+ * verify metadata against the specified pathname
+ */
+static int32_t crypt_open_cbk(call_frame_t *frame,
+			      void *cookie,
+			      xlator_t *this,
+			      int32_t op_ret,
+			      int32_t op_errno,
+			      fd_t *fd,
+			      dict_t *xdata)
+{
+	struct gf_flock lock = {0, };
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (local->fd->inode->ia_type == IA_IFLNK)
+		goto exit;
+	if (op_ret < 0)
+		goto exit;
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+	else if (local->custom_mtd){
+		local->xdata = dict_new();
+		if (!local->xdata) {
+			local->op_ret = -1;
+			local->op_errno = ENOMEM;
+			gf_log ("crypt", GF_LOG_ERROR,
+				"Can not get new dict for mtd string");
+			goto exit;
+		}
+	}
+	lock.l_len    = 0;
+        lock.l_start  = 0;
+        lock.l_type   = local->custom_mtd ? F_WRLCK : F_RDLCK;
+        lock.l_whence = SEEK_SET;
+
+	STACK_WIND(frame,
+		   crypt_open_finodelk_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->finodelk,
+		   this->name,
+		   fd,
+		   F_SETLKW,
+		   &lock,
+		   NULL);
+	return 0;
+ exit:
+	put_one_call_open(frame);
+	return 0;
+}
+
+static int32_t crypt_open(call_frame_t *frame,
+			  xlator_t *this,
+			  loc_t *loc,
+			  int32_t flags,
+			  fd_t *fd,
+			  dict_t *xdata)
+{
+	int32_t ret = ENOMEM;
+	crypt_local_t *local;
+
+	local = crypt_alloc_local(frame, this, GF_FOP_OPEN);
+	if (!local)
+		goto error;
+	local->loc = GF_CALLOC(1, sizeof(*loc), gf_crypt_mt_loc);
+	if (!local->loc) {
+		ret = ENOMEM;
+		goto error;
+	}
+	memset(local->loc, 0, sizeof(*local->loc));
+	ret = loc_copy(local->loc, loc);
+	if (ret) {
+		GF_FREE(local->loc);
+		goto error;
+	}
+	local->fd = fd_ref(fd);
+
+	ret = is_custom_mtd(xdata);
+	if (ret < 0) {
+		loc_wipe(local->loc);
+		GF_FREE(local->loc);
+		ret = EINVAL;
+		goto error;
+	}
+	local->custom_mtd = ret;
+
+	if ((flags & O_ACCMODE) == O_WRONLY)
+		/*
+		 * we can't open O_WRONLY, because
+		 * we need to do read-modify-write
+		 */
+		flags = (flags & ~O_ACCMODE) | O_RDWR;
+	/*
+	 * Make sure that out translated offsets
+	 * and counts won't be ignored
+	 */
+	flags &= ~O_APPEND;
+	get_one_call_nolock(frame);
+	STACK_WIND(frame,
+		   crypt_open_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->open,
+		   loc,
+		   flags,
+		   fd,
+		   xdata);
+	return 0;
+ error:
+	STACK_UNWIND_STRICT(open,
+			    frame,
+			    -1,
+			    ret,
+			    NULL,
+			    NULL);
+	return 0;
+}
+
+static int32_t init_inode_info_tail(struct crypt_inode_info *info,
+				    struct master_cipher_info *master)
+{
+	int32_t ret;
+	struct object_cipher_info *object = &info->cinfo;
+
+#if DEBUG_CRYPT
+	gf_log("crypt", GF_LOG_DEBUG, "Init inode info for object %s",
+	       uuid_utoa(info->oid));
+#endif
+	ret = data_cipher_algs[object->o_alg][object->o_mode].set_private(info,
+									master);
+	if (ret) {
+		gf_log("crypt", GF_LOG_ERROR, "Set private info failed");
+		return ret;
+	}
+	return 0;
+}
+
+/*
+ * Init inode info at ->create() time
+ */
+static void init_inode_info_create(struct crypt_inode_info *info,
+				   struct master_cipher_info *master,
+				   data_t *data)
+{
+	struct object_cipher_info *object;
+
+	info->nr_minor = CRYPT_XLATOR_ID;
+	memcpy(info->oid, data->data, data->len);
+
+	object = &info->cinfo;
+
+	object->o_alg        = master->m_alg;
+	object->o_mode       = master->m_mode;
+	object->o_block_bits = master->m_block_bits;
+	object->o_dkey_size  = master->m_dkey_size;
+}
+
+static void init_inode_info_head(struct crypt_inode_info *info, fd_t *fd)
+{
+	memcpy(info->oid, fd->inode->gfid, sizeof(uuid_t));
+}
+
+static int32_t crypt_create_done(call_frame_t *frame,
+				 void *cookie,
+				 xlator_t *this,
+				 int32_t op_ret,
+				 int32_t op_errno, dict_t *xdata)
+{
+	crypt_private_t *priv = this->private;
+	crypt_local_t *local = frame->local;
+	struct crypt_inode_info *info = local->info;
+	fd_t *local_fd = local->fd;
+	dict_t *local_xdata = local->xdata;
+	inode_t *local_inode = local->inode;
+
+	if (op_ret < 0) {
+		free_inode_info(info);
+		goto unwind;
+	}
+	op_errno = init_inode_info_tail(info, get_master_cinfo(priv));
+	if (op_errno) {
+		op_ret = -1;
+		free_inode_info(info);
+		goto unwind;
+	}
+	/*
+	 * FIXME: drop major subversion number
+	 */
+	op_ret = inode_ctx_put(local->fd->inode, this, (uint64_t)(long)info);
+	if (op_ret == -1) {
+		op_errno = EIO;
+		free_inode_info(info);
+		goto unwind;
+	}
+ unwind:
+	free_format(local);
+	STACK_UNWIND_STRICT(create,
+			    frame,
+			    op_ret,
+			    op_errno,
+			    local_fd,
+			    local_inode,
+			    &local->buf,
+			    &local->prebuf,
+			    &local->postbuf,
+			    local_xdata);
+	fd_unref(local_fd);
+	inode_unref(local_inode);
+	if (local_xdata)
+		dict_unref(local_xdata);
+	return 0;
+}
+
+static int crypt_create_tail(call_frame_t *frame,
+			     void *cookie,
+			     xlator_t *this,
+			     int32_t op_ret,
+			     int32_t op_errno,
+			     dict_t *xdata)
+{
+	struct gf_flock  lock  = {0, };
+	crypt_local_t *local = frame->local;
+	fd_t *local_fd = local->fd;
+	dict_t *local_xdata = local->xdata;
+	inode_t *local_inode = local->inode;
+
+	dict_unref(local->xattr);
+
+	if (op_ret < 0)
+		goto error;
+
+        lock.l_type   = F_UNLCK;
+        lock.l_whence = SEEK_SET;
+        lock.l_start  = 0;
+        lock.l_len    = 0;
+        lock.l_pid    = 0;
+
+	STACK_WIND(frame,
+		   crypt_create_done,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->finodelk,
+		   this->name,
+		   local->fd,
+		   F_SETLKW,
+		   &lock,
+		   NULL);
+	return 0;
+ error:
+	free_inode_info(local->info);
+	free_format(local);
+
+	STACK_UNWIND_STRICT(create,
+			    frame,
+			    op_ret,
+			    op_errno,
+			    local_fd,
+			    local_inode,
+			    &local->buf,
+			    &local->prebuf,
+			    &local->postbuf,
+			    local_xdata);
+
+	fd_unref(local_fd);
+	inode_unref(local_inode);
+	if (local_xdata)
+		dict_unref(local_xdata);
+	return 0;
+}
+
+static int32_t crypt_create_finodelk_cbk(call_frame_t *frame,
+					 void *cookie,
+					 xlator_t *this,
+					 int32_t op_ret,
+					 int32_t op_errno,
+					 dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	struct crypt_inode_info *info = local->info;
+
+	if (op_ret < 0)
+		goto error;
+
+	STACK_WIND(frame,
+		   crypt_create_tail,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fsetxattr,
+		   local->fd,
+		   local->xattr, /* CRYPTO_FORMAT_PREFIX */
+		   0,
+		   NULL);
+	return 0;
+ error: 
+	free_inode_info(info);
+	free_format(local);
+	fd_unref(local->fd);
+	dict_unref(local->xattr);
+	if (local->xdata)
+		dict_unref(local->xdata);
+
+	STACK_UNWIND_STRICT(create,
+			    frame,
+			    op_ret,
+			    op_errno,
+			    NULL,
+			    NULL,
+			    NULL,
+			    NULL,
+			    NULL,
+			    NULL);
+	return 0;
+}
+
+/*
+ * Create and store crypt-specific format on disk;
+ * Populate cache with private inode info
+ */
+static int32_t crypt_create_cbk(call_frame_t *frame,
+				void *cookie,
+				xlator_t *this,
+				int32_t op_ret,
+				int32_t op_errno,
+				fd_t *fd,
+				inode_t *inode,
+				struct iatt *buf,
+				struct iatt *preparent,
+				struct iatt *postparent,
+				dict_t *xdata)
+{
+	struct gf_flock lock = {0, };
+	crypt_local_t *local = frame->local;
+	struct crypt_inode_info *info = local->info;
+
+	if (op_ret < 0)
+		goto error;
+	if (xdata)
+		local->xdata = dict_ref(xdata);	
+	local->inode = inode_ref(inode);
+	local->buf = *buf;
+	local->prebuf = *preparent;
+	local->postbuf = *postparent;
+
+        lock.l_len    = 0;
+        lock.l_start  = 0;
+        lock.l_type   = F_WRLCK;
+        lock.l_whence = SEEK_SET;
+
+        STACK_WIND(frame,
+		   crypt_create_finodelk_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->finodelk,
+		   this->name,
+		   local->fd,
+		   F_SETLKW,
+		   &lock,
+		   NULL);	
+	return 0;
+ error:
+	free_inode_info(info);
+	free_format(local);
+	fd_unref(local->fd);
+	dict_unref(local->xattr);
+
+	STACK_UNWIND_STRICT(create,
+			    frame,
+			    op_ret,
+			    op_errno,
+			    NULL, NULL, NULL,
+			    NULL, NULL, NULL);
+	return 0;
+}
+
+static int32_t crypt_create(call_frame_t *frame,
+			    xlator_t *this,
+			    loc_t *loc,
+			    int32_t flags,
+			    mode_t mode,
+			    mode_t umask,
+			    fd_t *fd,
+			    dict_t *xdata)
+{
+	int ret;
+	data_t *data;
+	crypt_local_t *local;
+	crypt_private_t *priv;
+	struct master_cipher_info *master;
+	struct crypt_inode_info *info;
+
+	priv = this->private;
+	master = get_master_cinfo(priv);
+
+	if (master_alg_atomic(master)) {
+		/*
+		 * We can't open O_WRONLY, because we
+		 * need to do read-modify-write.
+		 */
+		if ((flags & O_ACCMODE) == O_WRONLY)
+			flags = (flags & ~O_ACCMODE) | O_RDWR;
+		/*
+		 * Make sure that out translated offsets
+		 * and counts won't be ignored
+		 */
+		flags &= ~O_APPEND;
+	}
+	local = crypt_alloc_local(frame, this, GF_FOP_CREATE);
+	if (!local) {
+		ret = ENOMEM;
+		goto error;
+	}
+	data = dict_get(xdata, "gfid-req");
+	if (!data) {
+		ret = EINVAL;
+		gf_log("crypt", GF_LOG_WARNING, "gfid not found");
+		goto error;
+	}
+	if (data->len != sizeof(uuid_t)) {
+		ret = EINVAL;
+		gf_log("crypt", GF_LOG_WARNING,
+		       "bad gfid size (%d), should be %d",
+		       (int)data->len, (int)sizeof(uuid_t));
+		goto error;
+	}
+	info = alloc_inode_info(local, loc);
+	if (!info){
+		ret = ENOMEM;
+		goto error;
+	}
+	/*
+	 * NOTE:
+	 * format has to be created BEFORE
+	 * proceeding to the untrusted server
+	 */
+	ret = alloc_format_create(local);
+	if (ret) {
+		free_inode_info(info);
+		goto error;
+	}
+	init_inode_info_create(info, master, data);
+
+	ret = create_format(local->format,
+			    loc,
+			    info,
+			    master);
+	if (ret) {
+		free_inode_info(info);
+		goto error;		
+	}
+	local->xattr = dict_new();
+	if (!local->xattr) {
+		free_inode_info(info);
+		free_format(local);	
+		goto error;
+	}
+	ret = dict_set_static_bin(local->xattr,
+				  CRYPTO_FORMAT_PREFIX,
+				  local->format,
+				  new_format_size());
+	if (ret) {
+		dict_unref(local->xattr);
+		free_inode_info(info);
+		free_format(local);
+		goto error;
+	}
+	ret = dict_set(local->xattr, FSIZE_XATTR_PREFIX, data_from_uint64(0));
+	if (ret) {
+		dict_unref(local->xattr);
+		free_inode_info(info);
+		free_format(local);
+		goto error;
+	}
+	local->fd = fd_ref(fd);
+
+	STACK_WIND(frame,
+		   crypt_create_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->create,
+		   loc,
+		   flags,
+		   mode,
+		   umask,
+		   fd,
+		   xdata);
+	return 0;
+ error:
+	gf_log("crypt", GF_LOG_WARNING, "can not create file");
+	STACK_UNWIND_STRICT(create,
+			    frame,
+			    -1,
+			    ret,
+			    NULL, NULL, NULL,
+			    NULL, NULL, NULL);
+	return 0;
+}
+
+/*
+ * FIXME: this should depends on the version of format string
+ */
+static int32_t filter_crypt_xattr(dict_t *dict,
+				  char *key, data_t *value, void *data)
+{
+        dict_del(dict, key);
+        return 0;
+}
+
+static int32_t crypt_fsetxattr(call_frame_t *frame,
+			       xlator_t *this,
+			       fd_t *fd,
+			       dict_t *dict,
+			       int32_t flags, dict_t *xdata)
+{
+	dict_foreach_fnmatch(dict, "trusted.glusterfs.crypt*",
+			     filter_crypt_xattr, NULL);
+	STACK_WIND(frame,
+		   default_fsetxattr_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fsetxattr,
+		   fd,
+		   dict,
+		   flags,
+		   xdata);
+	return 0;
+}
+
+/*
+ * TBD: verify file metadata before wind
+ */
+static int32_t crypt_setxattr(call_frame_t *frame,
+			      xlator_t *this,
+			      loc_t *loc,
+			      dict_t *dict,
+			      int32_t flags, dict_t *xdata)
+{
+	dict_foreach_fnmatch(dict, "trusted.glusterfs.crypt*",
+			     filter_crypt_xattr, NULL);
+	STACK_WIND(frame,
+		   default_setxattr_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->setxattr,
+		   loc,
+		   dict,
+		   flags,
+		   xdata);
+	return 0;
+}
+
+/*
+ * called as flush_cbk()
+ */
+static int32_t linkop_end(call_frame_t *frame,
+			  void *cookie,
+			  xlator_t *this,
+			  int32_t op_ret,
+			  int32_t op_errno,
+			  dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	linkop_unwind_handler_t unwind_fn;
+	unwind_fn = linkop_unwind_dispatch(local->fop);
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (op_ret < 0 &&
+	    op_errno == ENOENT &&
+	    local->loc->inode->ia_type == IA_IFLNK) {
+		local->op_ret = 0;
+		local->op_errno = 0;
+	}
+	unwind_fn(frame);
+	return 0;
+}
+
+/*
+ * unpin inode on the server
+ */
+static int32_t link_flush(call_frame_t *frame,
+			  void *cookie,
+			  xlator_t *this,
+			  int32_t op_ret,
+			  int32_t op_errno,
+			  inode_t *inode,
+			  struct iatt *buf,
+			  struct iatt *preparent,
+			  struct iatt *postparent, dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	if (op_ret < 0)
+		goto error;
+	if (local->xdata) {
+		dict_unref(local->xdata);
+		local->xdata = NULL;
+	}
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+	local->inode = inode_ref(inode);
+	local->buf = *buf;
+	local->prebuf = *preparent;
+	local->postbuf = *postparent;
+
+	STACK_WIND(frame,
+		   linkop_end,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->flush,
+		   local->fd,
+		   NULL);
+	return 0;
+ error:
+	local->op_ret = -1;
+	local->op_errno = op_errno;
+	link_unwind(frame);
+	return 0;
+}
+
+void link_unwind(call_frame_t *frame)
+{
+	crypt_local_t *local = frame->local;
+	dict_t *xdata;
+	dict_t *xattr;
+	inode_t *inode;
+
+	if (!local) {
+		STACK_UNWIND_STRICT(link,
+				    frame,
+				    -1,
+				    ENOMEM,
+				    NULL,
+				    NULL,
+				    NULL,
+				    NULL,
+				    NULL);
+		return;
+	}
+	xdata = local->xdata;
+	xattr = local->xattr;
+	inode = local->inode;
+
+	if (local->loc){
+		loc_wipe(local->loc);
+		GF_FREE(local->loc);
+	}
+	if (local->newloc) {
+		loc_wipe(local->newloc);
+		GF_FREE(local->newloc);
+	}
+	if (local->fd)
+		fd_unref(local->fd);
+	if (local->format)
+		GF_FREE(local->format);
+
+	STACK_UNWIND_STRICT(link,
+			    frame,
+			    local->op_ret,
+			    local->op_errno,
+			    inode,
+			    &local->buf,
+			    &local->prebuf,
+			    &local->postbuf,
+			    xdata);
+	if (xdata)
+		dict_unref(xdata);
+	if (xattr)
+		dict_unref(xattr);
+	if (inode)
+		inode_unref(inode);
+}
+
+void link_wind(call_frame_t *frame, xlator_t *this)
+{
+	crypt_local_t *local = frame->local;
+
+	STACK_WIND(frame,
+		   link_flush,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->link,
+		   local->loc,
+		   local->newloc,
+		   local->xdata);
+}
+
+/*
+ * unlink()
+ */
+static int32_t unlink_flush(call_frame_t *frame,
+			    void *cookie,
+			    xlator_t *this,
+			    int32_t op_ret,
+			    int32_t op_errno,
+			    struct iatt *preparent,
+			    struct iatt *postparent, dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	if (op_ret < 0)
+		goto error;
+	local->prebuf = *preparent;
+	local->postbuf = *postparent;
+	if (local->xdata) {
+		dict_unref(local->xdata);
+		local->xdata = NULL;
+	}
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+
+	STACK_WIND(frame,
+		   linkop_end,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->flush,
+		   local->fd,
+		   NULL);
+	return 0;
+ error:
+	local->op_ret = -1;
+	local->op_errno = op_errno;
+	unlink_unwind(frame);
+	return 0;
+}
+
+void unlink_unwind(call_frame_t *frame)
+{
+	crypt_local_t *local = frame->local;
+	dict_t *xdata;
+	dict_t *xattr;
+
+	if (!local) {
+		STACK_UNWIND_STRICT(unlink,
+				    frame,
+				    -1,
+				    ENOMEM,
+				    NULL,
+				    NULL,
+				    NULL);
+		return;
+	}
+	xdata = local->xdata;
+	xattr = local->xattr;
+	if (local->loc){
+		loc_wipe(local->loc);
+		GF_FREE(local->loc);
+	}
+	if (local->fd)
+		fd_unref(local->fd);
+	if (local->format)
+		GF_FREE(local->format);
+
+	STACK_UNWIND_STRICT(unlink,
+			    frame,
+			    local->op_ret,
+			    local->op_errno,
+			    &local->prebuf,
+			    &local->postbuf,
+			    xdata);
+	if (xdata)
+		dict_unref(xdata);
+	if (xattr)
+		dict_unref(xattr);
+}
+
+void unlink_wind(call_frame_t *frame, xlator_t *this)
+{
+	crypt_local_t *local = frame->local;
+
+	STACK_WIND(frame,
+		   unlink_flush,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->unlink,
+		   local->loc,
+		   local->flags,
+		   local->xdata);
+}
+
+void rename_unwind(call_frame_t *frame)
+{
+	crypt_local_t *local = frame->local;
+	dict_t *xdata;
+	dict_t *xattr;
+	struct iatt *prenewparent;
+	struct iatt *postnewparent;
+
+	if (!local) {
+		STACK_UNWIND_STRICT(rename,
+				    frame,
+				    -1,
+				    ENOMEM,
+				    NULL,
+				    NULL,
+				    NULL,
+				    NULL,
+				    NULL,
+				    NULL);	
+		return;
+	}
+	xdata = local->xdata;
+	xattr = local->xattr;
+	prenewparent = local->prenewparent;
+	postnewparent = local->postnewparent;	
+
+	if (local->loc){
+		loc_wipe(local->loc);
+		GF_FREE(local->loc);
+	}
+	if (local->newloc){
+		loc_wipe(local->newloc);
+		GF_FREE(local->newloc);
+	}
+	if (local->fd)
+		fd_unref(local->fd);
+	if (local->format)
+		GF_FREE(local->format);
+
+	STACK_UNWIND_STRICT(rename,
+			    frame,
+			    local->op_ret,
+			    local->op_errno,
+			    &local->buf,
+			    &local->prebuf,
+			    &local->postbuf,
+			    prenewparent,
+			    postnewparent,
+			    xdata);
+	if (xdata)
+		dict_unref(xdata);
+	if (xattr)
+		dict_unref(xattr);
+	if (prenewparent)
+		GF_FREE(prenewparent);
+	if (postnewparent)
+		GF_FREE(postnewparent);
+}
+
+/*
+ * called as flush_cbk()
+ */
+static int32_t rename_end(call_frame_t *frame,
+			  void *cookie,
+			  xlator_t *this,
+			  int32_t op_ret,
+			  int32_t op_errno,
+			  dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	rename_unwind(frame);
+	return 0;
+}
+
+static int32_t rename_flush(call_frame_t *frame,
+			    void *cookie,
+			    xlator_t *this,
+			    int32_t op_ret,
+			    int32_t op_errno,
+			    struct iatt *buf,
+			    struct iatt *preoldparent,
+			    struct iatt *postoldparent,
+			    struct iatt *prenewparent,
+			    struct iatt *postnewparent,
+			    dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	if (op_ret < 0)
+		goto error;
+	dict_unref(local->xdata);
+	local->xdata = NULL;
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+
+	local->buf = *buf;
+	local->prebuf = *preoldparent;
+	local->postbuf = *postoldparent;
+	if (prenewparent) {
+		local->prenewparent = GF_CALLOC(1, sizeof(*prenewparent),
+						gf_crypt_mt_iatt);
+		if (!local->prenewparent) {
+			op_errno = ENOMEM;
+			goto error;
+		}
+		*local->prenewparent = *prenewparent;
+	}
+	if (postnewparent) {
+		local->postnewparent = GF_CALLOC(1, sizeof(*postnewparent),
+						 gf_crypt_mt_iatt);
+		if (!local->postnewparent) {
+			op_errno = ENOMEM;
+			goto error;
+		}
+		*local->postnewparent = *postnewparent;
+	}
+	STACK_WIND(frame,
+		   rename_end,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->flush,
+		   local->fd,
+		   NULL);
+	return 0;
+ error:
+	local->op_ret = -1;
+	local->op_errno = op_errno;
+	rename_unwind(frame);
+	return 0;
+}
+
+void rename_wind(call_frame_t *frame, xlator_t *this)
+{
+	crypt_local_t *local = frame->local;
+
+	STACK_WIND(frame,
+		   rename_flush,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->rename,
+		   local->loc,
+		   local->newloc,
+		   local->xdata);
+}
+
+static int32_t __do_linkop(call_frame_t *frame,
+			   void *cookie,
+			   xlator_t *this,
+			   int32_t op_ret,
+			   int32_t op_errno, dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	linkop_wind_handler_t wind_fn;
+	linkop_unwind_handler_t unwind_fn;
+
+	wind_fn = linkop_wind_dispatch(local->fop);
+	unwind_fn = linkop_unwind_dispatch(local->fop);
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (op_ret >= 0)
+		wind_fn(frame, this);
+	else {
+		gf_log(this->name, GF_LOG_WARNING, "mtd unlock failed (%d)",
+		       op_errno);
+		unwind_fn(frame);
+	}
+	return 0;
+}
+
+static int32_t do_linkop(call_frame_t *frame,
+			 void *cookie,
+			 xlator_t *this,
+			 int32_t op_ret,
+			 int32_t op_errno,
+			 dict_t *xdata)
+{
+	struct gf_flock  lock  = {0, };
+	crypt_local_t *local = frame->local;
+	linkop_unwind_handler_t unwind_fn;
+
+	unwind_fn = linkop_unwind_dispatch(local->fop);
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if(op_ret < 0)
+		goto error;
+
+	lock.l_type   = F_UNLCK;
+        lock.l_whence = SEEK_SET;
+        lock.l_start  = 0;
+        lock.l_len    = 0;
+        lock.l_pid    = 0;
+
+	STACK_WIND(frame,
+		   __do_linkop,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->finodelk,
+		   this->name,
+		   local->fd,
+		   F_SETLKW,
+		   &lock,
+		   NULL);
+	return 0;
+ error:		
+	unwind_fn(frame);
+	return 0;
+}
+
+/*
+ * Update the metadata string (against the new pathname);
+ * submit the result
+ */
+static int32_t linkop_begin(call_frame_t *frame,
+			    void *cookie,
+			    xlator_t *this,
+			    int32_t op_ret,
+			    int32_t op_errno,
+			    fd_t *fd,
+			    dict_t *xdata)
+{
+	gf_boolean_t upload_info;
+	crypt_local_t *local = frame->local;
+	crypt_private_t *priv = this->private;
+	struct crypt_inode_info *info;
+	data_t *old_mtd;
+	uint32_t new_mtd_size;
+	uint64_t value = 0;
+	void (*unwind_fn)(call_frame_t *frame);
+	void (*wind_fn)(call_frame_t *frame, xlator_t *this);
+	mtd_op_t mop;
+
+	wind_fn = linkop_wind_dispatch(local->fop);
+	unwind_fn = linkop_unwind_dispatch(local->fop);
+	mop = linkop_mtdop_dispatch(local->fop);
+
+	if (local->fd->inode->ia_type == IA_IFLNK)
+		goto wind;
+	if (op_ret < 0)
+		/*
+		 * verification failed
+		 */
+		goto error;
+
+	old_mtd = dict_get(xdata, CRYPTO_FORMAT_PREFIX);
+	if (!old_mtd) {
+		op_errno = EIO;
+		gf_log (this->name, GF_LOG_DEBUG,
+			"Metadata string wasn't found");
+		goto error;
+	}
+	new_mtd_size = format_size(mop, old_mtd->len);
+	op_errno = alloc_format(local, new_mtd_size);
+	if (op_errno)
+		goto error;
+	/*
+	 * check for cached info 
+	 */
+	op_ret = inode_ctx_get(fd->inode, this, &value);
+	if (op_ret != -1) {
+		info = (struct crypt_inode_info *)(long)value;
+		if (info == NULL) {
+			gf_log (this->name, GF_LOG_WARNING,
+				"Inode info was not found");
+			op_errno = EINVAL;
+			goto error;
+		}
+		/*
+		 * info was found in the cache
+		 */
+		local->info = info;
+		upload_info = _gf_false;
+	}
+	else {
+		/*
+		 * info wasn't found in the cache;
+		 */
+		info = alloc_inode_info(local, local->loc);
+		if (!info)
+			goto error;
+		init_inode_info_head(info, fd);
+		local->info = info;
+		upload_info = _gf_true;
+	}
+	op_errno = open_format((unsigned char *)old_mtd->data,
+			       old_mtd->len,
+			       local->loc,
+			       info,
+			       get_master_cinfo(priv),
+			       local,
+			       upload_info);
+	if (op_errno)
+		goto error;
+	if (upload_info == _gf_true) {
+		op_errno = init_inode_info_tail(info,
+						get_master_cinfo(priv));
+		if (op_errno)
+			goto error;
+		op_errno = inode_ctx_put(fd->inode, this,
+					 (uint64_t)(long)(info));
+		if (op_errno == -1) {
+			op_errno = EIO;
+			goto error;
+		}
+	}
+	/*
+	 * update the format string (append/update/cup a MAC)
+	 */
+	op_errno = update_format(local->format,
+				 (unsigned char *)old_mtd->data,
+				 old_mtd->len,
+				 local->mac_idx,
+				 mop,
+				 local->newloc,
+				 info,
+				 get_master_cinfo(priv),
+				 local);
+	if (op_errno)
+		goto error;
+	/*
+	 * store the new format string on the server
+	 */
+	if (new_mtd_size) {
+		op_errno = dict_set_static_bin(local->xattr,
+					       CRYPTO_FORMAT_PREFIX,
+					       local->format,
+					       new_mtd_size);
+		if (op_errno)
+			goto error;
+	}
+	STACK_WIND(frame,
+		   do_linkop,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->setxattr,
+		   local->loc,
+		   local->xattr,
+		   0,
+		   NULL);
+	return 0;
+ wind:
+	wind_fn(frame, this);
+	return 0;
+ error:
+	local->op_ret = -1;
+	local->op_errno = op_errno;
+	unwind_fn(frame);
+	return 0;
+}
+
+static int32_t linkop_grab_local(call_frame_t *frame,
+				 xlator_t *this,
+				 loc_t *oldloc,
+				 loc_t *newloc,
+				 int flags, dict_t *xdata,
+				 glusterfs_fop_t op)
+{
+	int32_t ret = ENOMEM;
+	fd_t *fd;
+	crypt_local_t *local;
+
+	local = crypt_alloc_local(frame, this, op);
+	if (!local)
+		goto error;
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+
+	fd = fd_create(oldloc->inode, frame->root->pid);
+	if (!fd) {
+		gf_log(this->name, GF_LOG_ERROR, "Can not create fd");
+		goto error;
+	}
+	local->fd = fd;
+	local->flags = flags;
+	local->loc = GF_CALLOC(1, sizeof(*oldloc), gf_crypt_mt_loc);
+	if (!local->loc)
+		goto error;
+	memset(local->loc, 0, sizeof(*local->loc));
+	ret = loc_copy(local->loc, oldloc);
+	if (ret) {
+		GF_FREE(local->loc);
+		local->loc = NULL;
+		goto error;
+	}
+	if (newloc) {
+		local->newloc = GF_CALLOC(1, sizeof(*newloc), gf_crypt_mt_loc);
+		if (!local->newloc) {
+			loc_wipe(local->loc);
+			GF_FREE(local->loc);
+			goto error;
+		}
+		memset(local->newloc, 0, sizeof(*local->newloc));
+		ret = loc_copy(local->newloc, newloc);
+		if (ret) {
+			loc_wipe(local->loc);
+			GF_FREE(local->loc);
+			GF_FREE(local->newloc);
+			goto error;
+		}
+	}
+	local->xattr = dict_new();
+	if (!local->xattr) {
+		gf_log(this->name, GF_LOG_ERROR, "Can not create dict");
+		ret = ENOMEM;
+		goto error;
+	}
+	return 0;
+
+error:
+        if (local) {
+                if (local->xdata)
+                        dict_unref(local->xdata);
+                if (local->fd)
+                        fd_unref(local->fd);
+                local->fd = 0;
+                local->loc = NULL;
+                local->newloc = NULL;
+                local->op_ret = -1;
+                local->op_errno = ret;
+        }
+
+        return ret;
+}
+
+/*
+ * read and verify locked metadata against the old pathname (via open);
+ * update the metadata string in accordance with the new pathname;
+ * submit modified metadata;
+ * wind;
+ */
+static int32_t linkop(call_frame_t *frame,
+		      xlator_t *this,
+		      loc_t *oldloc,
+		      loc_t *newloc,
+		      int flags,
+		      dict_t *xdata,
+		      glusterfs_fop_t op)
+{
+	int32_t ret;
+	dict_t *dict;
+	crypt_local_t *local;
+	void (*unwind_fn)(call_frame_t *frame);
+
+	unwind_fn = linkop_unwind_dispatch(op);
+
+	ret = linkop_grab_local(frame, this, oldloc, newloc, flags, xdata, op);
+	local = frame->local;
+	if (ret)
+		goto error;
+	dict = dict_new();
+	if (!dict) {
+		gf_log(this->name, GF_LOG_ERROR, "Can not create dict");
+		ret = ENOMEM;
+		goto error;
+	}
+	/*
+	 * Set a message to crypt_open() that we need
+	 * locked metadata string.
+	 * All link operations (link, unlink, rename)
+	 * need write lock
+	 */
+	msgflags_set_mtd_wlock(&local->msgflags);
+	ret = dict_set_static_bin(dict,
+				  MSGFLAGS_PREFIX,
+				  &local->msgflags,
+				  sizeof(local->msgflags));
+	if (ret) {
+		gf_log(this->name, GF_LOG_ERROR, "Can not set dict");
+		dict_unref(dict);
+		goto error;
+	}
+	/*
+	 * verify metadata against the old pathname
+	 * and retrieve locked metadata string
+	 */
+	STACK_WIND(frame,
+		   linkop_begin,
+		   this,
+		   this->fops->open, /* crypt_open() */
+		   oldloc,
+		   O_RDWR,
+		   local->fd,
+		   dict);
+	dict_unref(dict);
+	return 0;
+ error:
+	local->op_ret = -1;
+	local->op_errno = ret;
+	unwind_fn(frame);
+	return 0;
+}
+
+static int32_t crypt_link(call_frame_t *frame, xlator_t *this,
+			  loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+{
+	return linkop(frame, this, oldloc, newloc, 0, xdata, GF_FOP_LINK);
+}
+
+static int32_t crypt_unlink(call_frame_t *frame, xlator_t *this,
+			    loc_t *loc, int flags, dict_t *xdata)
+{
+	return linkop(frame, this, loc, NULL, flags, xdata, GF_FOP_UNLINK);
+}
+
+static int32_t crypt_rename(call_frame_t *frame, xlator_t *this,
+			    loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+{
+	return linkop(frame, this, oldloc, newloc, 0, xdata, GF_FOP_RENAME);
+}
+
+static void put_one_call_open(call_frame_t *frame)
+{
+	crypt_local_t *local = frame->local;
+	if (put_one_call(local)) {
+		fd_t *fd = local->fd;
+		loc_t *loc = local->loc;
+		dict_t *xdata = local->xdata;
+
+		STACK_UNWIND_STRICT(open,
+				    frame,
+				    local->op_ret,
+				    local->op_errno,
+				    fd,
+				    xdata);
+		fd_unref(fd);
+		if (xdata)
+			dict_unref(xdata);
+		loc_wipe(loc);
+		GF_FREE(loc);
+	}
+}
+
+static int32_t __crypt_readv_done(call_frame_t *frame,
+				  void *cookie,
+				  xlator_t *this,
+				  int32_t op_ret,
+				  int32_t op_errno, dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	fd_t *local_fd = local->fd;
+	dict_t *local_xdata = local->xdata;
+	/* read deals with data configs only */
+	struct iovec *avec = local->data_conf.avec;
+	char **pool = local->data_conf.pool;
+	int blocks_in_pool = local->data_conf.blocks_in_pool;
+	struct iobref *iobref = local->iobref;
+	struct iobref *iobref_data = local->iobref_data;
+
+	if (op_ret < 0) {
+		gf_log(this->name, GF_LOG_WARNING,
+		       "readv unlock failed (%d)", op_errno);
+		if (local->op_ret >= 0) {
+			local->op_ret = op_ret;
+			local->op_errno = op_errno;
+		}
+	}
+	dump_plain_text(local, avec);
+
+	gf_log("crypt", GF_LOG_DEBUG,
+	       "readv: ret_to_user: %d, iovec len: %d, ia_size: %llu",
+	       (int)(local->rw_count > 0 ? local->rw_count : local->op_ret),
+	       (int)(local->rw_count > 0 ? iovec_get_size(avec, local->data_conf.acount) : 0),
+	       (unsigned long long)local->buf.ia_size);
+
+	STACK_UNWIND_STRICT(readv,
+			    frame,
+			    local->rw_count > 0 ? local->rw_count : local->op_ret,
+			    local->op_errno,
+			    avec,
+			    avec ? local->data_conf.acount : 0,
+			    &local->buf,
+			    local->iobref,
+			    local_xdata);
+
+	free_avec(avec, pool, blocks_in_pool);
+	fd_unref(local_fd);
+	if (local_xdata)
+		dict_unref(local_xdata);
+	if (iobref)
+		iobref_unref(iobref);
+	if (iobref_data)
+		iobref_unref(iobref_data);
+	return 0;
+}
+
+static void crypt_readv_done(call_frame_t *frame, xlator_t *this)
+{
+	if (parent_is_crypt_xlator(frame, this))
+		/*
+		 *  don't unlock (it will be done by the parent)
+		 */
+		__crypt_readv_done(frame, NULL, this, 0, 0, NULL);
+	else {
+		crypt_local_t *local = frame->local;
+		struct gf_flock  lock  = {0, };
+
+		lock.l_type   = F_UNLCK;
+		lock.l_whence = SEEK_SET;
+		lock.l_start  = 0;
+		lock.l_len    = 0;
+		lock.l_pid    = 0;
+
+		STACK_WIND(frame,
+			   __crypt_readv_done,
+			   FIRST_CHILD(this),
+			   FIRST_CHILD(this)->fops->finodelk,
+			   this->name,
+			   local->fd,
+			   F_SETLKW,
+			   &lock,
+			   NULL);
+	}
+}
+
+static void put_one_call_readv(call_frame_t *frame, xlator_t *this)
+{
+	crypt_local_t *local = frame->local;
+	if (put_one_call(local))
+		crypt_readv_done(frame, this);
+}
+
+static int32_t __crypt_writev_done(call_frame_t *frame,
+				   void *cookie,
+				   xlator_t *this,
+				   int32_t op_ret,
+				   int32_t op_errno, dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	fd_t *local_fd = local->fd;
+	dict_t *local_xdata = local->xdata;
+	int32_t ret_to_user;
+
+	if (local->xattr)
+		dict_unref(local->xattr);
+	/*
+	 * Calculate amout of butes to be returned
+	 * to user. We need to subtract paddings that
+	 * have been written as a part of atom.
+	 */
+	/*
+	 * subtract head padding
+	 */
+	if (local->rw_count == 0)
+		/*
+		 * Nothing has been written, it must be an error
+		 */
+		ret_to_user = local->op_ret;
+	else if (local->rw_count <= local->data_conf.off_in_head) {
+		gf_log("crypt", GF_LOG_WARNING, "Incomplete write");
+		ret_to_user = 0;
+	}
+	else
+		ret_to_user = local->rw_count -
+			local->data_conf.off_in_head;
+	/*
+	 * subtract tail padding
+	 */
+	if (ret_to_user > local->data_conf.orig_size)
+		ret_to_user = local->data_conf.orig_size;
+
+	if (local->iobref)
+		iobref_unref(local->iobref);
+	if (local->iobref_data)
+		iobref_unref(local->iobref_data);
+	free_avec_data(local);
+	free_avec_hole(local);
+
+	gf_log("crypt", GF_LOG_DEBUG,
+	       "writev: ret_to_user: %d", ret_to_user);
+
+	STACK_UNWIND_STRICT(writev,
+			    frame,
+			    ret_to_user,
+			    local->op_errno,
+			    &local->prebuf,
+			    &local->postbuf,
+			    local_xdata);
+	fd_unref(local_fd);
+	if (local_xdata)
+		dict_unref(local_xdata);
+	return 0;
+}
+
+static int32_t crypt_writev_done(call_frame_t *frame,
+				 void *cookie,
+				 xlator_t *this,
+				 int32_t op_ret,
+				 int32_t op_errno,
+				 dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	if (op_ret < 0)
+		gf_log("crypt", GF_LOG_WARNING, "can not update file size");
+
+	if (parent_is_crypt_xlator(frame, this))
+		/*
+		 * don't unlock (it will be done by the parent)
+		 */
+		__crypt_writev_done(frame, NULL, this, 0, 0, NULL);
+	else {
+		struct gf_flock  lock  = {0, };
+
+		lock.l_type   = F_UNLCK;
+		lock.l_whence = SEEK_SET;
+		lock.l_start  = 0;
+		lock.l_len    = 0;
+		lock.l_pid    = 0;
+
+		STACK_WIND(frame,
+			   __crypt_writev_done,
+			   FIRST_CHILD(this),
+			   FIRST_CHILD(this)->fops->finodelk,
+			   this->name,
+			   local->fd,
+			   F_SETLKW,
+			   &lock,
+			   NULL);
+	}
+	return 0;
+}
+
+static void put_one_call_writev(call_frame_t *frame, xlator_t *this)
+{
+	crypt_local_t *local = frame->local;
+	if (put_one_call(local)) {
+		if (local->update_disk_file_size) {
+			int32_t ret;
+			/*
+			 * update file size, unlock the file and unwind
+			 */
+			ret = dict_set(local->xattr,
+				       FSIZE_XATTR_PREFIX,
+				       data_from_uint64(local->cur_file_size));
+			if (ret) {
+				gf_log("crypt", GF_LOG_WARNING,
+				       "can not set key to update file size");
+				crypt_writev_done(frame, NULL,
+						  this, 0, 0, NULL);
+				return;
+			}
+			gf_log("crypt", GF_LOG_DEBUG,
+			       "Updating disk file size to %llu",
+			       (unsigned long long)local->cur_file_size);
+			STACK_WIND(frame,
+				   crypt_writev_done,
+				   FIRST_CHILD(this),
+				   FIRST_CHILD(this)->fops->fsetxattr,
+				   local->fd,
+				   local->xattr, /* CRYPTO_FORMAT_PREFIX */
+				   0,
+				   NULL);
+		}
+		else
+			crypt_writev_done(frame, NULL, this, 0, 0, NULL);
+	}
+}
+
+static int32_t __crypt_ftruncate_done(call_frame_t *frame,
+				      void *cookie,
+				      xlator_t *this,
+				      int32_t op_ret,
+				      int32_t op_errno, dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	fd_t *local_fd = local->fd;
+	dict_t *local_xdata = local->xdata;
+	char *iobase = local->vec.iov_base;
+
+	if (op_ret < 0) {
+		gf_log(this->name, GF_LOG_WARNING,
+		       "ftruncate unlock failed (%d)", op_errno);
+		if (local->op_ret >= 0) {
+			local->op_ret = op_ret;
+			local->op_errno = op_errno;
+		}
+	}
+	if (local->iobref_data)
+		iobref_unref(local->iobref_data);
+	free_avec_data(local);
+	free_avec_hole(local);
+
+	gf_log("crypt", GF_LOG_DEBUG,
+	       "ftruncate, return to user: presize=%llu, postsize=%llu",
+	       (unsigned long long)local->prebuf.ia_size,
+	       (unsigned long long)local->postbuf.ia_size);
+
+	STACK_UNWIND_STRICT(ftruncate,
+			    frame,
+			    local->op_ret < 0 ? -1 : 0,
+			    local->op_errno,
+			    &local->prebuf,
+			    &local->postbuf,
+			    local_xdata);
+	fd_unref(local_fd);
+	if (local_xdata)
+		dict_unref(local_xdata);
+	if (iobase)
+		GF_FREE(iobase);
+	return 0;
+}
+
+static int32_t crypt_ftruncate_done(call_frame_t *frame,
+				    void *cookie,
+				    xlator_t *this,
+				    int32_t op_ret,
+				    int32_t op_errno,
+				    dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	struct gf_flock  lock  = {0, };
+
+	dict_unref(local->xattr);
+	if (op_ret < 0)
+		gf_log("crypt", GF_LOG_WARNING, "can not update file size");
+
+	lock.l_type   = F_UNLCK;
+	lock.l_whence = SEEK_SET;
+	lock.l_start  = 0;
+	lock.l_len    = 0;
+	lock.l_pid    = 0;
+
+	STACK_WIND(frame,
+		   __crypt_ftruncate_done,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->finodelk,
+		   this->name,
+		   local->fd,
+		   F_SETLKW,
+		   &lock,
+		   NULL);
+	return 0;
+}
+
+static void put_one_call_ftruncate(call_frame_t *frame, xlator_t *this)
+{
+	crypt_local_t *local = frame->local;
+	if (put_one_call(local)) {
+		if (local->update_disk_file_size) {
+			int32_t ret;
+			/*
+			 * update file size, unlock the file and unwind
+			 */
+			ret = dict_set(local->xattr,
+				       FSIZE_XATTR_PREFIX,
+				       data_from_uint64(local->cur_file_size));
+			if (ret) {
+				gf_log("crypt", GF_LOG_WARNING,
+				       "can not set key to update file size");
+				crypt_ftruncate_done(frame, NULL,
+						     this, 0, 0, NULL);
+				return;
+			}
+			gf_log("crypt", GF_LOG_DEBUG,
+			       "Updating disk file size to %llu",
+			       (unsigned long long)local->cur_file_size);
+			STACK_WIND(frame,
+				   crypt_ftruncate_done,
+				   FIRST_CHILD(this),
+				   FIRST_CHILD(this)->fops->fsetxattr,
+				   local->fd,
+				   local->xattr, /* CRYPTO_FORMAT_PREFIX */
+				   0,
+				   NULL);
+		}
+		else
+			crypt_ftruncate_done(frame, NULL, this, 0, 0, NULL);
+	}
+}
+
+/*
+ * load regular file size for some FOPs
+ */
+static int32_t load_file_size(call_frame_t *frame,
+			      void *cookie,
+			      xlator_t *this,
+			      int32_t op_ret,
+			      int32_t op_errno,
+			      dict_t *dict,
+			      dict_t *xdata)
+{
+	data_t *data;
+	crypt_local_t *local = frame->local;
+
+	dict_t *local_xdata = local->xdata;
+	inode_t *local_inode = local->inode;
+
+	if (op_ret < 0)
+		goto unwind;
+	/*
+	 * load regular file size
+	 */
+	data = dict_get(dict, FSIZE_XATTR_PREFIX);
+	if (!data) {
+		if (local->xdata)
+			dict_unref(local->xdata);
+		gf_log("crypt", GF_LOG_WARNING, "Regular file size not found");
+		op_ret = -1;
+		op_errno = EIO;
+		goto unwind;
+	}
+	local->buf.ia_size = data_to_uint64(data);
+
+	gf_log(this->name, GF_LOG_DEBUG,
+	       "FOP %d: Translate regular file to %llu",
+	       local->fop,
+	       (unsigned long long)local->buf.ia_size);
+ unwind:	
+	if (local->fd)
+		fd_unref(local->fd);
+	if (local->loc) {
+		loc_wipe(local->loc);
+		GF_FREE(local->loc);
+	}
+	switch (local->fop) {
+	case GF_FOP_FSTAT:
+		STACK_UNWIND_STRICT(fstat,
+				    frame,
+				    op_ret,
+				    op_errno,
+				    op_ret >= 0 ? &local->buf : NULL,
+				    local->xdata);
+		break;
+	case GF_FOP_STAT:
+		STACK_UNWIND_STRICT(stat,
+				    frame,
+				    op_ret,
+				    op_errno,
+				    op_ret >= 0 ? &local->buf : NULL,
+				    local->xdata);
+		break;
+	case GF_FOP_LOOKUP:
+		STACK_UNWIND_STRICT(lookup,
+				    frame,
+				    op_ret,
+				    op_errno,
+				    op_ret >= 0 ? local->inode : NULL,
+				    op_ret >= 0 ? &local->buf : NULL,				    
+				    local->xdata,
+				    op_ret >= 0 ? &local->postbuf : NULL);
+		break;
+	case GF_FOP_READ:
+		STACK_UNWIND_STRICT(readv,
+				    frame,
+				    op_ret,
+				    op_errno,
+				    NULL,
+				    0,
+				    op_ret >= 0 ? &local->buf : NULL,
+				    NULL,
+				    NULL);
+		break;
+	default:
+		gf_log(this->name, GF_LOG_WARNING,
+		       "Improper file operation %d", local->fop);
+	}
+	if (local_xdata)
+		dict_unref(local_xdata);
+	if (local_inode)
+		inode_unref(local_inode);
+	return 0;
+}
+
+static int32_t crypt_stat_common_cbk(call_frame_t *frame,
+				     void *cookie,
+				     xlator_t *this,
+				     int32_t op_ret,
+				     int32_t op_errno,
+				     struct iatt *buf, dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	if (op_ret < 0)
+		goto unwind;
+	if (!IA_ISREG(buf->ia_type))
+		goto unwind;
+
+	local->buf = *buf;
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+
+	switch (local->fop) {
+	case GF_FOP_FSTAT:
+		STACK_WIND(frame,
+			   load_file_size,
+			   FIRST_CHILD(this),
+			   FIRST_CHILD(this)->fops->fgetxattr,
+			   local->fd,
+			   FSIZE_XATTR_PREFIX,
+			   NULL);
+		break;
+	case GF_FOP_STAT:
+		STACK_WIND(frame,
+			   load_file_size,
+			   FIRST_CHILD(this),
+			   FIRST_CHILD(this)->fops->getxattr,
+			   local->loc,
+			   FSIZE_XATTR_PREFIX,
+			   NULL);
+		break;
+	default:
+		gf_log (this->name, GF_LOG_WARNING,
+			"Improper file operation %d", local->fop);
+	}
+	return 0;
+ unwind:
+	if (local->fd)
+		fd_unref(local->fd);
+	if (local->loc) {
+		loc_wipe(local->loc);
+		GF_FREE(local->loc);
+	}
+	switch (local->fop) {
+	case GF_FOP_FSTAT:
+		STACK_UNWIND_STRICT(fstat,
+				    frame,
+				    op_ret,
+				    op_errno,
+				    op_ret >= 0 ? buf : NULL,
+				    op_ret >= 0 ? xdata : NULL);
+		break;
+	case GF_FOP_STAT:
+		STACK_UNWIND_STRICT(stat,
+				    frame,
+				    op_ret,
+				    op_errno,
+				    op_ret >= 0 ? buf : NULL,
+				    op_ret >= 0 ? xdata : NULL);
+		break;
+	default:
+		gf_log (this->name, GF_LOG_WARNING,
+			"Improper file operation %d", local->fop);
+	}
+	return 0;
+}
+
+static int32_t crypt_fstat(call_frame_t *frame,
+			   xlator_t *this,
+			   fd_t *fd, dict_t *xdata)
+{
+	crypt_local_t *local;
+
+	local = crypt_alloc_local(frame, this, GF_FOP_FSTAT);
+	if (!local)
+		goto error;
+	local->fd = fd_ref(fd);
+	STACK_WIND(frame,
+		   crypt_stat_common_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fstat,
+		   fd,
+		   xdata);	
+	return 0;
+ error:
+	STACK_UNWIND_STRICT(fstat,
+			    frame,
+			    -1,
+			    ENOMEM,
+			    NULL,
+			    NULL);
+	return 0;
+}
+
+static int32_t crypt_stat(call_frame_t *frame,
+			  xlator_t *this,
+			  loc_t *loc, dict_t *xdata)
+{
+	int32_t ret;
+	crypt_local_t *local;
+
+	local = crypt_alloc_local(frame, this, GF_FOP_STAT);
+	if (!local)
+		goto error;
+	local->loc = GF_CALLOC(1, sizeof(*loc), gf_crypt_mt_loc);
+	if (!local->loc)
+		goto error;
+	memset(local->loc, 0, sizeof(*local->loc));
+	ret = loc_copy(local->loc, loc);
+	if (ret) {
+		GF_FREE(local->loc);
+		goto error;
+	}
+	STACK_WIND(frame,
+		   crypt_stat_common_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->stat,
+		   loc,
+		   xdata);	
+	return 0;
+ error:
+	STACK_UNWIND_STRICT(stat,
+			    frame,
+			    -1,
+			    ENOMEM,
+			    NULL,
+			    NULL);
+	return 0;
+}
+
+static int32_t crypt_lookup_cbk(call_frame_t *frame,
+				void *cookie,
+				xlator_t *this,
+				int32_t op_ret,
+				int32_t op_errno,
+				inode_t *inode,
+				struct iatt *buf, dict_t *xdata,
+				struct iatt *postparent)
+{
+	crypt_local_t *local = frame->local;
+
+	if (op_ret < 0)
+		goto unwind;
+	if (!IA_ISREG(buf->ia_type))
+		goto unwind;
+
+	local->inode = inode_ref(inode);
+	local->buf = *buf;
+	local->postbuf = *postparent;
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+	uuid_copy(local->loc->gfid, buf->ia_gfid);
+
+	STACK_WIND(frame,
+		   load_file_size,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->getxattr,
+		   local->loc,
+		   FSIZE_XATTR_PREFIX,
+		   NULL);
+	return 0;
+ unwind:
+	loc_wipe(local->loc);
+	GF_FREE(local->loc);
+	STACK_UNWIND_STRICT(lookup,
+			    frame,
+			    op_ret,
+			    op_errno,
+			    inode,
+			    buf,
+			    xdata,
+			    postparent);
+	return 0;
+}
+
+static int32_t crypt_lookup(call_frame_t *frame,
+			    xlator_t *this,
+			    loc_t *loc, dict_t *xdata)
+{
+	int32_t ret;
+	crypt_local_t *local;
+
+	local = crypt_alloc_local(frame, this, GF_FOP_LOOKUP);
+	if (!local)
+		goto error;
+	local->loc = GF_CALLOC(1, sizeof(*loc), gf_crypt_mt_loc);
+	if (!local->loc)
+		goto error;
+	memset(local->loc, 0, sizeof(*local->loc));
+	ret = loc_copy(local->loc, loc);
+	if (ret) {
+		GF_FREE(local->loc);
+		goto error;
+	}
+	gf_log(this->name, GF_LOG_DEBUG, "Lookup %s", loc->path);
+	STACK_WIND(frame,
+		   crypt_lookup_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->lookup,
+		   loc,
+		   xdata);
+	return 0;
+ error:
+	STACK_UNWIND_STRICT(lookup,
+			    frame,
+			    -1,
+			    ENOMEM,
+			    NULL,
+			    NULL,
+			    NULL,
+			    NULL);
+	return 0;
+}
+
+/*
+ * for every regular directory entry find its real file size
+ * and update stat's buf properly
+ */
+static int32_t crypt_readdirp_cbk(call_frame_t *frame,
+				  void *cookie,
+				  xlator_t *this,
+				  int32_t op_ret,
+				  int32_t op_errno,
+				  gf_dirent_t *entries, dict_t *xdata)
+{
+	gf_dirent_t *entry = NULL;
+
+	if (op_ret < 0)
+		goto unwind;
+
+	list_for_each_entry (entry, (&entries->list), list) {
+		data_t *data;
+
+		if (!IA_ISREG(entry->d_stat.ia_type))
+			continue;
+		data = dict_get(entry->dict, FSIZE_XATTR_PREFIX);
+		if (!data){
+			gf_log("crypt", GF_LOG_WARNING,
+			       "Regular file size of direntry not found");
+			op_errno = EIO;
+			op_ret = -1;
+			break;
+		}
+		entry->d_stat.ia_size = data_to_uint64(data);
+	}
+ unwind:
+	STACK_UNWIND_STRICT(readdirp, frame, op_ret, op_errno, entries, xdata);
+	return 0;
+}
+
+/*
+ * ->readdirp() fills in-core inodes, so we need to set proper
+ * file sizes for all directory entries of the parent @fd.
+ * Actual updates take place in ->crypt_readdirp_cbk()
+ */
+static int32_t crypt_readdirp(call_frame_t *frame, xlator_t *this,
+			      fd_t *fd, size_t size, off_t offset,
+			      dict_t *xdata)
+{
+	int32_t ret = ENOMEM;
+
+	if (!xdata) {
+		xdata = dict_new();
+		if (!xdata)
+			goto error;
+	}
+	else
+		dict_ref(xdata);
+	/*
+	 * make sure that we'll have real file sizes at ->readdirp_cbk()
+	 */
+	ret = dict_set(xdata, FSIZE_XATTR_PREFIX, data_from_uint64(0));
+	if (ret) {
+		dict_unref(xdata);
+		goto error;
+	}
+	STACK_WIND(frame,
+		   crypt_readdirp_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->readdirp,
+		   fd,
+		   size,
+		   offset,
+		   xdata);
+	dict_unref(xdata);
+	return 0;
+ error:
+	STACK_UNWIND_STRICT(readdirp, frame, -1, ret, NULL, NULL);
+	return 0;
+}
+
+static int32_t crypt_access(call_frame_t *frame,
+			    xlator_t *this,
+			    loc_t *loc,
+			    int32_t mask, dict_t *xdata)
+{
+	gf_log(this->name, GF_LOG_WARNING,
+	       "NFS mounts of encrypted volumes are unsupported");
+	STACK_UNWIND_STRICT(access, frame, -1, EPERM, NULL);
+	return 0;
+}
+
+int32_t master_set_block_size (xlator_t *this, crypt_private_t *priv,
+			       dict_t *options)
+{
+	uint64_t block_size = 0;
+	struct master_cipher_info *master = get_master_cinfo(priv);
+
+	if (options != NULL)
+		GF_OPTION_RECONF("block-size", block_size, options,
+				 size, error);
+	else
+		GF_OPTION_INIT("block-size", block_size, size, error);
+
+	switch (block_size) {
+	case 512:
+		master->m_block_bits = 9;
+		break;
+	case 1024:
+		master->m_block_bits = 10;
+		break;
+	case 2048:
+		master->m_block_bits = 11;
+		break;
+	case 4096:
+		master->m_block_bits = 12;
+		break;
+	default:
+		gf_log("crypt", GF_LOG_ERROR,
+		       "FATAL: unsupported block size %llu",
+		       (unsigned long long)block_size);
+		goto error;
+	}
+	return 0;
+ error:
+	return -1;
+}
+
+int32_t master_set_alg(xlator_t *this, crypt_private_t *priv)
+{
+	struct master_cipher_info *master = get_master_cinfo(priv);
+	master->m_alg = AES_CIPHER_ALG;
+	return 0;
+}
+
+int32_t master_set_mode(xlator_t *this, crypt_private_t *priv)
+{
+	struct master_cipher_info *master = get_master_cinfo(priv);
+	master->m_mode = XTS_CIPHER_MODE;
+	return 0;
+}
+
+/*
+ * set key size in bits to the master info
+ * Pre-conditions: cipher mode in the master info is uptodate.
+ */
+static int master_set_data_key_size (xlator_t *this, crypt_private_t *priv,
+				     dict_t *options)
+{
+	int32_t ret;