789 files changed, 424364 insertions, 179319 deletions

diff --git a/xlators/Makefile.am b/xlators/Makefile.am
index 4c94f5e44c1..ef20cbb64fa 100644
--- a/xlators/Makefile.am
+++ b/xlators/Makefile.am
@@ -1,3 +1,13 @@
-SUBDIRS = cluster storage protocol performance debug features encryption mount nfs mgmt
+if BUILD_GNFS
+  GNFS_DIR = nfs
+endif
+
+DIST_SUBDIRS = cluster storage protocol performance debug features \
+          mount nfs mgmt system playground meta
+
+SUBDIRS = cluster storage protocol performance debug features \
+          mount ${GNFS_DIR} mgmt system playground meta
+
+EXTRA_DIST = xlator.sym
 
 CLEANFILES =
diff --git a/xlators/bindings/Makefile.am b/xlators/bindings/Makefile.am
deleted file mode 100644
index f7766580257..00000000000
--- a/xlators/bindings/Makefile.am
+++ /dev/null
@@ -1 +0,0 @@
-SUBDIRS = $(BINDINGS_SUBDIRS)
diff --git a/xlators/bindings/python/src/Makefile.am b/xlators/bindings/python/src/Makefile.am
deleted file mode 100644
index c0b9141c667..00000000000
--- a/xlators/bindings/python/src/Makefile.am
+++ /dev/null
@@ -1,19 +0,0 @@
-
-xlator_PROGRAMS = python.so
-
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/bindings
-
-python_PYTHON = gluster.py glustertypes.py glusterstack.py
-
-pythondir = $(xlatordir)/python
-
-python_so_SOURCES = python.c
-
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall \
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles \
-	$(PYTHON_CPPLAGS) -DGLUSTER_PYTHON_PATH=\"$(pythondir)\"
-
-AM_LDFLAGS = $(PYTHON_LDFLAGS)
-
-CLEANFILES = 
-
diff --git a/xlators/bindings/python/src/gluster.py b/xlators/bindings/python/src/gluster.py
deleted file mode 100644
index ee0eb131011..00000000000
--- a/xlators/bindings/python/src/gluster.py
+++ /dev/null
@@ -1,47 +0,0 @@
-#   Copyright (c) 2007 Chris AtLee <chris@atlee.ca>
-#   This file is part of GlusterFS.
-#
-#   GlusterFS is free software; you can redistribute it and/or modify
-#   it under the terms of the GNU General Public License as published
-#   by the Free Software Foundation; either version 3 of the License,
-#   or (at your option) any later version.
-#
-#   GlusterFS is distributed in the hope that it will be useful, but
-#   WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-#   General Public License for more details.
-#
-#   You should have received a copy of the GNU General Public License
-#   along with this program.  If not, see
-#   <http://www.gnu.org/licenses/>.
-from ctypes import *
-from glustertypes import *
-from glusterstack import *
-import sys
-import inspect
-
-libglusterfs = CDLL("libglusterfs.so")
-_gf_log = libglusterfs._gf_log
-_gf_log.restype = c_int32
-_gf_log.argtypes = [c_char_p, c_char_p, c_char_p, c_int32, c_int, c_char_p]
-
-gf_log_loglevel = c_int.in_dll(libglusterfs, "gf_log_loglevel")
-
-GF_LOG_NONE = 0
-GF_LOG_CRITICAL = 1
-GF_LOG_ERROR = 2
-GF_LOG_WARNING = 3
-GF_LOG_DEBUG = 4
-
-def gf_log(module, level, fmt, *params):
-    if level <= gf_log_loglevel:
-        frame = sys._getframe(1)
-        _gf_log(module, frame.f_code.co_filename, frame.f_code.co_name,
-                frame.f_lineno, level, fmt, *params)
-
-class ComplexTranslator(object):
-    def __init__(self, xlator):
-        self.xlator = xlator_t.from_address(xlator)
-
-    def __getattr__(self, item):
-        return getattr(self.xlator, item)
diff --git a/xlators/bindings/python/src/glusterstack.py b/xlators/bindings/python/src/glusterstack.py
deleted file mode 100644
index ba24c81652e..00000000000
--- a/xlators/bindings/python/src/glusterstack.py
+++ /dev/null
@@ -1,55 +0,0 @@
-#   Copyright (c) 2007 Chris AtLee <chris@atlee.ca>
-#   This file is part of GlusterFS.
-#
-#   GlusterFS is free software; you can redistribute it and/or modify
-#   it under the terms of the GNU General Public License as published
-#   by the Free Software Foundation; either version 3 of the License,
-#   or (at your option) any later version.
-#
-#   GlusterFS is distributed in the hope that it will be useful, but
-#   WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-#   General Public License for more details.
-#
-#   You should have received a copy of the GNU General Public License
-#   along with this program.  If not, see
-#   <http://www.gnu.org/licenses/>.
-from ctypes import *
-from glustertypes import *
-
-libc = CDLL("libc.so.6")
-calloc = libc.calloc
-calloc.argtypes = [c_int, c_int]
-calloc.restype = c_void_p
-
-# TODO: Can these be done in C somehow?
-def stack_wind(frame, rfn, obj, fn, *params):
-    """Frame is a frame object"""
-    _new = cast(calloc(1, sizeof(call_frame_t)), POINTER(call_frame_t))
-    _new[0].root = frame.root
-    _new[0].next = frame.root[0].frames.next
-    _new[0].prev = pointer(frame.root[0].frames)
-    if frame.root[0].frames.next:
-        frame.root[0].frames.next[0].prev = _new
-    frame.root[0].frames.next = _new
-    _new[0].this = obj
-    # TODO: Type checking like tmp_cbk?
-    _new[0].ret = rfn
-    _new[0].parent = pointer(frame)
-    _new[0].cookie = cast(_new, c_void_p)
-    # TODO: Initialize lock
-    #_new.lock.init()
-    frame.ref_count += 1
-    fn(_new, obj, *params)
-
-def stack_unwind(frame, *params):
-    """Frame is a frame object"""
-    fn = frame[0].ret
-    parent = frame[0].parent[0]
-    parent.ref_count -= 1
-
-    op_ret = params[0]
-    op_err = params[1]
-    params = params[2:]
-    fn(parent, call_frame_t.from_address(frame[0].cookie), parent.this,
-            op_ret, op_err, *params)
diff --git a/xlators/bindings/python/src/glustertypes.py b/xlators/bindings/python/src/glustertypes.py
deleted file mode 100644
index e9069d07c72..00000000000
--- a/xlators/bindings/python/src/glustertypes.py
+++ /dev/null
@@ -1,167 +0,0 @@
-#   Copyright (c) 2007 Chris AtLee <chris@atlee.ca>
-#   This file is part of GlusterFS.
-#
-#   GlusterFS is free software; you can redistribute it and/or modify
-#   it under the terms of the GNU General Public License as published
-#   by the Free Software Foundation; either version 3 of the License,
-#   or (at your option) any later version.
-#
-#   GlusterFS is distributed in the hope that it will be useful, but
-#   WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-#   General Public License for more details.
-#
-#   You should have received a copy of the GNU General Public License
-#   along with this program.  If not, see
-#   <http://www.gnu.org/licenses/>.
-from ctypes import *
-import collections
-
-#
-# Forward declaration of some gluster types
-#
-class call_frame_t(Structure):
-    pass
-
-class call_ctx_t(Structure):
-    pass
-
-class call_pool_t(Structure):
-    pass
-
-class xlator_t(Structure):
-    def _getFirstChild(self):
-        return self.children[0].xlator
-    firstChild = property(_getFirstChild)
-
-class xlator_list_t(Structure):
-    pass
-
-class xlator_fops(Structure):
-    pass
-
-class xlator_mops(Structure):
-    pass
-
-class glusterfs_ctx_t(Structure):
-    pass
-
-class list_head(Structure):
-    pass
-
-class dict_t(Structure):
-    pass
-
-class inode_table_t(Structure):
-    pass
-
-class fd_t(Structure):
-    pass
-
-class iovec(Structure):
-    _fields_ = [
-            ("iov_base", c_void_p),
-            ("iov_len", c_size_t),
-            ]
-
-    def __init__(self, s):
-        self.iov_base = cast(c_char_p(s), c_void_p)
-        self.iov_len = len(s)
-
-    def getBytes(self):
-        return string_at(self.iov_base, self.iov_len)
-
-# This is a pthread_spinlock_t
-# TODO: what happens to volatile-ness?
-gf_lock_t = c_int
-
-uid_t = c_uint32
-gid_t = c_uint32
-pid_t = c_int32
-
-off_t = c_int64
-
-#
-# Function pointer types
-#
-ret_fn_t = CFUNCTYPE(c_int32, POINTER(call_frame_t), POINTER(call_frame_t),
-                              POINTER(xlator_t), c_int32, c_int32)
-
-fini_fn_t = CFUNCTYPE(None, POINTER(xlator_t))
-init_fn_t = CFUNCTYPE(c_int32, POINTER(xlator_t))
-event_notify_fn_t = CFUNCTYPE(c_int32, POINTER(xlator_t), c_int32, c_void_p)
-
-list_head._fields_ = [
-        ("next", POINTER(list_head)),
-        ("prev", POINTER(list_head)),
-        ]
-
-call_frame_t._fields_ = [
-        ("root", POINTER(call_ctx_t)),
-        ("parent", POINTER(call_frame_t)),
-        ("next", POINTER(call_frame_t)),
-        ("prev", POINTER(call_frame_t)),
-        ("local", c_void_p),
-        ("this", POINTER(xlator_t)),
-        ("ret", ret_fn_t),
-        ("ref_count", c_int32),
-        ("lock", gf_lock_t),
-        ("cookie", c_void_p),
-        ("op", c_int32),
-        ("type", c_int8),
-        ]
-
-call_ctx_t._fields_ = [
-        ("all_frames", list_head),
-        ("trans", c_void_p),
-        ("pool", call_pool_t),
-        ("unique", c_uint64),
-        ("state", c_void_p),
-        ("uid", uid_t),
-        ("gid", gid_t),
-        ("pid", pid_t),
-        ("frames", call_frame_t),
-        ("req_refs", POINTER(dict_t)),
-        ("rsp_refs", POINTER(dict_t)),
-        ]
-
-xlator_t._fields_ = [
-        ("name", c_char_p),
-        ("type", c_char_p),
-        ("next", POINTER(xlator_t)),
-        ("prev", POINTER(xlator_t)),
-        ("parent", POINTER(xlator_t)),
-        ("children", POINTER(xlator_list_t)),
-        ("fops", POINTER(xlator_fops)),
-        ("mops", POINTER(xlator_mops)),
-        ("fini", fini_fn_t),
-        ("init", init_fn_t),
-        ("notify", event_notify_fn_t),
-        ("options", POINTER(dict_t)),
-        ("ctx", POINTER(glusterfs_ctx_t)),
-        ("itable", POINTER(inode_table_t)),
-        ("ready", c_char),
-        ("private", c_void_p),
-        ]
-
-xlator_list_t._fields_ = [
-        ("xlator", POINTER(xlator_t)),
-        ("next", POINTER(xlator_list_t)),
-        ]
-
-fop_functions = collections.defaultdict(lambda: c_void_p)
-fop_function_names = ['lookup', 'forget', 'stat', 'fstat', 'chmod', 'fchmod',
-        'chown', 'fchown', 'truncate', 'ftruncate', 'utimens', 'access',
-        'readlink', 'mknod', 'mkdir', 'unlink', 'rmdir', 'symlink',
-        'rename', 'link', 'create', 'open', 'readv', 'writev', 'flush',
-        'close', 'fsync', 'opendir', 'readdir', 'closedir', 'fsyncdir',
-        'statfs', 'setxattr', 'getxattr', 'removexattr', 'lk', 'writedir',
-        # TODO: Call backs?
-        ]
-
-fop_writev_t = CFUNCTYPE(c_int32, POINTER(call_frame_t), POINTER(xlator_t),
-                                  POINTER(fd_t), POINTER(iovec), c_int32,
-                                  off_t)
-
-fop_functions['writev'] = fop_writev_t
-xlator_fops._fields_ = [(f, fop_functions[f]) for f in fop_function_names]
diff --git a/xlators/bindings/python/src/python.c b/xlators/bindings/python/src/python.c
deleted file mode 100644
index 3310a211513..00000000000
--- a/xlators/bindings/python/src/python.c
+++ /dev/null
@@ -1,232 +0,0 @@
-/*
-   Copyright (c) 2007-2010 Chris AtLee <chris@atlee.ca>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-#include <Python.h>
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "xlator.h"
-#include "logging.h"
-#include "defaults.h"
-
-typedef struct
-{
-    char        *scriptname;
-    PyObject    *pXlator;
-    PyObject    *pScriptModule;
-    PyObject    *pGlusterModule;
-    PyThreadState *pInterp;
-
-    PyObject    *pFrameType, *pVectorType, *pFdType;
-} python_private_t;
-
-int32_t
-python_writev (call_frame_t *frame,
-              xlator_t *this,
-              fd_t *fd,
-              struct iovec *vector,
-              int32_t count, 
-              off_t offset)
-{
-  python_private_t *priv = (python_private_t *)this->private;
-  gf_log("python", GF_LOG_DEBUG, "In writev");
-  if (PyObject_HasAttrString(priv->pXlator, "writev"))
-  {
-
-      PyObject *retval = PyObject_CallMethod(priv->pXlator, "writev",
-              "O O O i l",
-              PyObject_CallMethod(priv->pFrameType, "from_address", "O&", PyLong_FromVoidPtr, frame),
-              PyObject_CallMethod(priv->pFdType, "from_address", "O&", PyLong_FromVoidPtr, fd),
-              PyObject_CallMethod(priv->pVectorType, "from_address", "O&", PyLong_FromVoidPtr, vector),
-              count,
-              offset);
-      if (PyErr_Occurred())
-      {
-          PyErr_Print();
-      }
-      Py_XDECREF(retval);
-  }
-  else
-  {
-      return default_writev(frame, this, fd, vector, count, offset);
-  }
-  return 0;
-}
-
-struct xlator_fops fops = {
-    .writev       = python_writev
-};
-
-static PyObject *
-AnonModule_FromFile (const char* fname)
-{
-    // Get the builtins
-    PyThreadState* pThread = PyThreadState_Get();
-    PyObject *pBuiltins = pThread->interp->builtins;
-
-    if (PyErr_Occurred())
-    {
-        PyErr_Print();
-        return NULL;
-    }
-
-    // Create a new dictionary for running code in
-    PyObject *pModuleDict = PyDict_New();
-    PyDict_SetItemString(pModuleDict, "__builtins__", pBuiltins);
-    Py_INCREF(pBuiltins);
-
-    // Run the file in the new context
-    FILE* fp = fopen(fname, "r");
-    PyRun_File(fp, fname, Py_file_input, pModuleDict, pModuleDict);
-    fclose(fp);
-    if (PyErr_Occurred())
-    {
-        PyErr_Print();
-        Py_DECREF(pModuleDict);
-        Py_DECREF(pBuiltins);
-        return NULL;
-    }
-
-    // Create an object to hold the new context
-    PyRun_String("class ModuleWrapper(object):\n\tpass\n", Py_single_input, pModuleDict, pModuleDict);
-    if (PyErr_Occurred())
-    {
-        PyErr_Print();
-        Py_DECREF(pModuleDict);
-        Py_DECREF(pBuiltins);
-        return NULL;
-    }
-    PyObject *pModule = PyRun_String("ModuleWrapper()", Py_eval_input, pModuleDict, pModuleDict);
-    if (PyErr_Occurred())
-    {
-        PyErr_Print();
-        Py_DECREF(pModuleDict);
-        Py_DECREF(pBuiltins);
-        Py_XDECREF(pModule);
-        return NULL;
-    }
-
-    // Set the new context's dictionary to the one we used to run the code
-    // inside
-    PyObject_SetAttrString(pModule, "__dict__", pModuleDict);
-    if (PyErr_Occurred())
-    {
-        PyErr_Print();
-        Py_DECREF(pModuleDict);
-        Py_DECREF(pBuiltins);
-        Py_DECREF(pModule);
-        return NULL;
-    }
-
-    return pModule;
-}
-
-int32_t
-init (xlator_t *this)
-{
-  // This is ok to call more than once per process
-  Py_InitializeEx(0);
-
-  if (!this->children) {
-    gf_log ("python", GF_LOG_ERROR, 
-            "FATAL: python should have exactly one child");
-    return -1;
-  }
-
-  python_private_t *priv = CALLOC (sizeof (python_private_t), 1);
-  ERR_ABORT (priv);
-
-  data_t *scriptname = dict_get (this->options, "scriptname");
-  if (scriptname) {
-      priv->scriptname = data_to_str(scriptname);
-  } else {
-      gf_log("python", GF_LOG_ERROR,
-              "FATAL: python requires the scriptname parameter");
-      return -1;
-  }
-
-  priv->pInterp = Py_NewInterpreter();
-    
-  // Adjust python's path
-  PyObject *syspath = PySys_GetObject("path");
-  PyObject *path = PyString_FromString(GLUSTER_PYTHON_PATH);
-  PyList_Append(syspath, path);
-  Py_DECREF(path);
-
-  gf_log("python", GF_LOG_DEBUG,
-          "Loading gluster module");
-
-  priv->pGlusterModule = PyImport_ImportModule("gluster");
-  if (PyErr_Occurred())
-  {
-      PyErr_Print();
-      return -1;
-  }
-
-  priv->pFrameType = PyObject_GetAttrString(priv->pGlusterModule, "call_frame_t");
-  priv->pFdType = PyObject_GetAttrString(priv->pGlusterModule, "fd_t");
-  priv->pVectorType = PyObject_GetAttrString(priv->pGlusterModule, "iovec");
-
-  gf_log("python", GF_LOG_DEBUG, "Loading script...%s", priv->scriptname);
-  
-  priv->pScriptModule = AnonModule_FromFile(priv->scriptname);
-  if (!priv->pScriptModule || PyErr_Occurred())
-  {
-      gf_log("python", GF_LOG_ERROR, "Error loading %s", priv->scriptname);
-      PyErr_Print();
-      return -1;
-  }
-
-  if (!PyObject_HasAttrString(priv->pScriptModule, "xlator"))
-  {
-      gf_log("python", GF_LOG_ERROR, "%s does not have a xlator attribute", priv->scriptname);
-      return -1;
-  }
-  gf_log("python", GF_LOG_DEBUG, "Instantiating translator");
-  priv->pXlator = PyObject_CallMethod(priv->pScriptModule, "xlator", "O&",
-          PyLong_FromVoidPtr, this);
-  if (PyErr_Occurred() || !priv->pXlator)
-  {
-      PyErr_Print();
-      return -1;
-  }
-
-  this->private = priv;
-
-  gf_log ("python", GF_LOG_DEBUG, "python xlator loaded");
-  return 0;
-}
-
-void 
-fini (xlator_t *this)
-{
-  python_private_t *priv = (python_private_t*)(this->private);
-  Py_DECREF(priv->pXlator);
-  Py_DECREF(priv->pScriptModule);
-  Py_DECREF(priv->pGlusterModule);
-  Py_DECREF(priv->pFrameType);
-  Py_DECREF(priv->pFdType);
-  Py_DECREF(priv->pVectorType);
-  Py_EndInterpreter(priv->pInterp);
-  return;
-}
diff --git a/xlators/bindings/python/src/testxlator.py b/xlators/bindings/python/src/testxlator.py
deleted file mode 100644
index 507455c856a..00000000000
--- a/xlators/bindings/python/src/testxlator.py
+++ /dev/null
@@ -1,56 +0,0 @@
-#   Copyright (c) 2007 Chris AtLee <chris@atlee.ca>
-#   This file is part of GlusterFS.
-#
-#   GlusterFS is free software; you can redistribute it and/or modify
-#   it under the terms of the GNU General Public License as published
-#   by the Free Software Foundation; either version 3 of the License,
-#   or (at your option) any later version.
-#
-#   GlusterFS is distributed in the hope that it will be useful, but
-#   WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-#   General Public License for more details.
-#
-#   You should have received a copy of the GNU General Public License
-#   along with this program.  If not, see
-#   <http://www.gnu.org/licenses/>.
-
-"""
-This is a test translator written in python.
-
-Important things to note:
-    This file must be import-able from glusterfsd.  This probably means
-    setting PYTHONPATH to where this file is located.
-
-    This file must have a top-level xlator class object that will be
-    used to instantiate individual translators.
-"""
-from gluster import *
-
-class MyXlator(ComplexTranslator):
-    name = "MyXlator"
-    def writev_cbk(self, frame, cookie, op_ret, op_errno, buf):
-        stack_unwind(frame, op_ret, op_errno, buf)
-        return 0
-
-    def writev(self, frame, fd, vector, count, offset):
-        gf_log(self.name, GF_LOG_WARNING, "writev %i bytes", vector.iov_len)
-        # TODO: Use cookie to pass this to writev_cbk
-        old_count = vector.iov_len
-
-        data = vector.getBytes().encode("zlib")
-
-        vector = iovec(data)
-        gf_log(self.name, GF_LOG_WARNING, "writev %i bytes", vector.iov_len)
-
-        @ret_fn_t
-        def rfn(frame, prev, this, op_ret, op_errno, *params):
-            if len(params) == 0:
-                params = [0]
-            return self.writev_cbk(frame, prev, old_count, op_errno, *params)
-
-        stack_wind(frame, rfn, self.firstChild,
-                self.firstChild[0].fops[0].writev, fd, vector, count, offset)
-        return 0
-
-xlator = MyXlator
diff --git a/xlators/cluster/Makefile.am b/xlators/cluster/Makefile.am
index 0990822a7d3..8e067d5ab58 100644
--- a/xlators/cluster/Makefile.am
+++ b/xlators/cluster/Makefile.am
@@ -1,3 +1,3 @@
-SUBDIRS = stripe afr dht
+SUBDIRS = afr dht ec
 
 CLEANFILES = 
diff --git a/xlators/cluster/afr/src/Makefile.am b/xlators/cluster/afr/src/Makefile.am
index e192b599bf4..610819b28fc 100644
--- a/xlators/cluster/afr/src/Makefile.am
+++ b/xlators/cluster/afr/src/Makefile.am
@@ -1,27 +1,35 @@
-xlator_LTLIBRARIES = afr.la pump.la
+xlator_LTLIBRARIES = afr.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/cluster
 
-afr_common_source = afr-dir-read.c afr-dir-write.c afr-inode-read.c afr-inode-write.c afr-open.c afr-transaction.c afr-self-heal-data.c afr-self-heal-common.c afr-self-heal-metadata.c afr-self-heal-entry.c afr-self-heal-algorithm.c afr-lk-common.c $(top_builddir)/xlators/lib/src/libxlator.c
+afr_common_source = afr-dir-read.c afr-dir-write.c afr-inode-read.c \
+	afr-inode-write.c afr-open.c afr-transaction.c afr-lk-common.c \
+	afr-read-txn.c \
+	$(top_builddir)/xlators/lib/src/libxlator.c
 
-afr_la_LDFLAGS = -module -avoidversion
-afr_la_SOURCES = $(afr_common_source) afr.c
+AFR_SELFHEAL_SOURCES = afr-self-heal-common.c afr-self-heal-data.c \
+	afr-self-heal-entry.c afr-self-heal-metadata.c afr-self-heald.c \
+	afr-self-heal-name.c
+
+afr_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
+afr_la_SOURCES = $(afr_common_source) $(AFR_SELFHEAL_SOURCES) afr.c
 afr_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-pump_la_LDFLAGS = -module -avoidversion
-pump_la_SOURCES =  $(afr_common_source) pump.c
-pump_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+noinst_HEADERS = afr.h afr-transaction.h afr-inode-write.h afr-inode-read.h \
+	afr-dir-read.h afr-dir-write.h afr-self-heal.h afr-mem-types.h \
+	afr-common.c afr-self-heald.h \
+	$(top_builddir)/xlators/lib/src/libxlator.h afr-messages.h
 
-noinst_HEADERS = afr.h afr-transaction.h afr-inode-write.h afr-inode-read.h afr-dir-read.h afr-dir-write.h afr-self-heal.h afr-self-heal-common.h afr-self-heal-algorithm.h pump.h afr-mem-types.h afr-common.c $(top_builddir)/xlators/lib/src/libxlator.h
+AM_CPPFLAGS = $(GF_CPPFLAGS) \
+	-I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/xlators/lib/src \
+	-I$(top_srcdir)/rpc/rpc-lib/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
-	    -I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/contrib/md5 -shared -nostartfiles $(GF_CFLAGS) \
-	    -I$(top_srcdir)/xlators/lib/src
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES =
 
 uninstall-local:
 	rm -f $(DESTDIR)$(xlatordir)/replicate.so
-	rm -f $(DESTDIR)$(xlatordir)/pump.so
 
 install-data-hook:
 	ln -sf afr.so $(DESTDIR)$(xlatordir)/replicate.so
diff --git a/xlators/cluster/afr/src/afr-common.c b/xlators/cluster/afr/src/afr-common.c
index 81f01c125f8..032ab5c8001 100644
--- a/xlators/cluster/afr/src/afr-common.c
+++ b/xlators/cluster/afr/src/afr-common.c
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2007-2010 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include <libgen.h>
@@ -24,28 +15,20 @@
 #include <stdlib.h>
 #include <signal.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
+#include <glusterfs/glusterfs.h>
 #include "afr.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "stack.h"
-#include "list.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include "compat-errno.h"
-#include "compat.h"
-#include "byte-order.h"
-#include "statedump.h"
-
-#include "fd.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/hashfn.h>
+#include <glusterfs/list.h>
+#include <glusterfs/call-stub.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/byte-order.h>
+#include <glusterfs/statedump.h>
+#include <glusterfs/events.h>
+#include <glusterfs/upcall-utils.h>
 
 #include "afr-inode-read.h"
 #include "afr-inode-write.h"
@@ -53,2563 +36,7843 @@
 #include "afr-dir-write.h"
 #include "afr-transaction.h"
 #include "afr-self-heal.h"
-#include "afr-self-heal-common.h"
-#include "pump.h"
-
-#define AFR_ICTX_OPENDIR_DONE_MASK     0x0000000200000000ULL
-#define AFR_ICTX_SPLIT_BRAIN_MASK      0x0000000100000000ULL
-#define AFR_ICTX_READ_CHILD_MASK       0x00000000FFFFFFFFULL
+#include "afr-self-heald.h"
+#include "afr-messages.h"
 
 int32_t
-afr_set_dict_gfid (dict_t *dict, uuid_t gfid)
+afr_quorum_errno(afr_private_t *priv)
 {
-        int ret = 0;
-
-        GF_ASSERT (gfid);
+    return ENOTCONN;
+}
 
-        ret = dict_set_static_bin (dict, "gfid-req", gfid, 16);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_DEBUG, "gfid set failed");
+gf_boolean_t
+afr_is_private_directory(afr_private_t *priv, uuid_t pargfid, const char *name,
+                         pid_t pid)
+{
+    if (!__is_root_gfid(pargfid)) {
+        return _gf_false;
+    }
+
+    if (strcmp(name, GF_REPLICATE_TRASH_DIR) == 0) {
+        /*For backward compatibility /.landfill is private*/
+        return _gf_true;
+    }
+
+    if (pid == GF_CLIENT_PID_GSYNCD) {
+        /*geo-rep needs to create/sync private directory on slave because
+         * it appears in changelog*/
+        return _gf_false;
+    }
+
+    if (pid == GF_CLIENT_PID_GLFS_HEAL || pid == GF_CLIENT_PID_SELF_HEALD) {
+        if (strcmp(name, priv->anon_inode_name) == 0) {
+            /* anonymous-inode dir is private*/
+            return _gf_true;
+        }
+    } else {
+        if (strncmp(name, AFR_ANON_DIR_PREFIX, strlen(AFR_ANON_DIR_PREFIX)) ==
+            0) {
+            /* anonymous-inode dir prefix is private for geo-rep to work*/
+            return _gf_true;
+        }
+    }
 
-        return ret;
+    return _gf_false;
 }
 
-uint64_t
-afr_is_split_brain (xlator_t *this, inode_t *inode)
+void
+afr_fill_success_replies(afr_local_t *local, afr_private_t *priv,
+                         unsigned char *replies)
 {
-        int ret = 0;
+    int i = 0;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->replies[i].valid && local->replies[i].op_ret == 0) {
+            replies[i] = 1;
+        } else {
+            replies[i] = 0;
+        }
+    }
+}
 
-        uint64_t ctx         = 0;
-        uint64_t split_brain = 0;
+int
+afr_fav_child_reset_sink_xattrs(void *opaque);
 
-        VALIDATE_OR_GOTO (inode, out);
+int
+afr_fav_child_reset_sink_xattrs_cbk(int ret, call_frame_t *frame, void *opaque);
 
-        LOCK (&inode->lock);
-        {
-                ret = __inode_ctx_get (inode, this, &ctx);
+static void
+afr_discover_done(call_frame_t *frame, xlator_t *this);
 
-                if (ret < 0)
-                        goto unlock;
+int
+afr_dom_lock_acquire_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int op_ret, int op_errno, dict_t *xdata)
+{
+    afr_local_t *local = frame->local;
+    afr_private_t *priv = this->private;
+    int i = (long)cookie;
+
+    local->cont.lk.dom_lock_op_ret[i] = op_ret;
+    local->cont.lk.dom_lock_op_errno[i] = op_errno;
+    if (op_ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, AFR_MSG_LK_HEAL_DOM,
+               "%s: Failed to acquire %s on %s",
+               uuid_utoa(local->fd->inode->gfid), AFR_LK_HEAL_DOM,
+               priv->children[i]->name);
+    } else {
+        local->cont.lk.dom_locked_nodes[i] = 1;
+    }
+
+    syncbarrier_wake(&local->barrier);
+
+    return 0;
+}
 
-                split_brain = ctx & AFR_ICTX_SPLIT_BRAIN_MASK;
+int
+afr_dom_lock_acquire(call_frame_t *frame)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    struct gf_flock flock = {
+        0,
+    };
+    int i = 0;
+
+    priv = frame->this->private;
+    local = frame->local;
+    local->cont.lk.dom_locked_nodes = GF_CALLOC(
+        priv->child_count, sizeof(*local->cont.lk.locked_nodes),
+        gf_afr_mt_char);
+    if (!local->cont.lk.dom_locked_nodes) {
+        return -ENOMEM;
+    }
+    local->cont.lk.dom_lock_op_ret = GF_CALLOC(
+        priv->child_count, sizeof(*local->cont.lk.dom_lock_op_ret),
+        gf_afr_mt_int32_t);
+    if (!local->cont.lk.dom_lock_op_ret) {
+        return -ENOMEM; /* CALLOC'd members are freed in afr_local_cleanup. */
+    }
+    local->cont.lk.dom_lock_op_errno = GF_CALLOC(
+        priv->child_count, sizeof(*local->cont.lk.dom_lock_op_errno),
+        gf_afr_mt_int32_t);
+    if (!local->cont.lk.dom_lock_op_errno) {
+        return -ENOMEM; /* CALLOC'd members are freed in afr_local_cleanup. */
+    }
+    flock.l_type = F_WRLCK;
+
+    AFR_ONALL(frame, afr_dom_lock_acquire_cbk, finodelk, AFR_LK_HEAL_DOM,
+              local->fd, F_SETLK, &flock, NULL);
+
+    if (!afr_has_quorum(local->cont.lk.dom_locked_nodes, frame->this, NULL))
+        goto blocking_lock;
+
+    /*If any of the bricks returned EAGAIN, we still need blocking locks.*/
+    if (AFR_COUNT(local->cont.lk.dom_locked_nodes, priv->child_count) !=
+        priv->child_count) {
+        for (i = 0; i < priv->child_count; i++) {
+            if (local->cont.lk.dom_lock_op_ret[i] == -1 &&
+                local->cont.lk.dom_lock_op_errno[i] == EAGAIN)
+                goto blocking_lock;
         }
-unlock:
-        UNLOCK (&inode->lock);
+    }
 
-out:
-        return split_brain;
+    return 0;
+
+blocking_lock:
+    afr_dom_lock_release(frame);
+    AFR_ONALL(frame, afr_dom_lock_acquire_cbk, finodelk, AFR_LK_HEAL_DOM,
+              local->fd, F_SETLKW, &flock, NULL);
+    if (!afr_has_quorum(local->cont.lk.dom_locked_nodes, frame->this, NULL)) {
+        afr_dom_lock_release(frame);
+        return -afr_quorum_errno(priv);
+    }
+
+    return 0;
 }
 
+int
+afr_dom_lock_release_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int op_ret, int op_errno, dict_t *xdata)
+{
+    afr_local_t *local = frame->local;
+    afr_private_t *priv = this->private;
+    int i = (long)cookie;
+
+    if (op_ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, AFR_MSG_LK_HEAL_DOM,
+               "%s: Failed to release %s on %s", local->loc.path,
+               AFR_LK_HEAL_DOM, priv->children[i]->name);
+    }
+    local->cont.lk.dom_locked_nodes[i] = 0;
+
+    syncbarrier_wake(&local->barrier);
+
+    return 0;
+}
 
 void
-afr_set_split_brain (xlator_t *this, inode_t *inode, gf_boolean_t set)
+afr_dom_lock_release(call_frame_t *frame)
 {
-        uint64_t ctx = 0;
-        int      ret = 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    unsigned char *locked_on = NULL;
+    struct gf_flock flock = {
+        0,
+    };
+
+    local = frame->local;
+    priv = frame->this->private;
+    locked_on = local->cont.lk.dom_locked_nodes;
+    if (AFR_COUNT(locked_on, priv->child_count) == 0)
+        return;
+    flock.l_type = F_UNLCK;
 
-        VALIDATE_OR_GOTO (inode, out);
+    AFR_ONLIST(locked_on, frame, afr_dom_lock_release_cbk, finodelk,
+               AFR_LK_HEAL_DOM, local->fd, F_SETLK, &flock, NULL);
 
-        LOCK (&inode->lock);
-        {
-                ret = __inode_ctx_get (inode, this, &ctx);
+    return;
+}
 
-                if (ret < 0) {
-                        ctx = 0;
-                }
+static void
+afr_lk_heal_info_cleanup(afr_lk_heal_info_t *info)
+{
+    if (!info)
+        return;
+    if (info->xdata_req)
+        dict_unref(info->xdata_req);
+    if (info->fd)
+        fd_unref(info->fd);
+    GF_FREE(info->locked_nodes);
+    GF_FREE(info->child_up_event_gen);
+    GF_FREE(info->child_down_event_gen);
+    GF_FREE(info);
+}
 
-                if (set) {
-                        ctx = (~AFR_ICTX_SPLIT_BRAIN_MASK & ctx)
-                                | (0xFFFFFFFFFFFFFFFFULL & AFR_ICTX_SPLIT_BRAIN_MASK);
-                } else {
-                        ctx = (~AFR_ICTX_SPLIT_BRAIN_MASK & ctx);
-                }
-                __inode_ctx_put (inode, this, ctx);
+static int
+afr_add_lock_to_saved_locks(call_frame_t *frame, xlator_t *this)
+{
+    afr_private_t *priv = this->private;
+    afr_local_t *local = frame->local;
+    afr_lk_heal_info_t *info = NULL;
+    afr_fd_ctx_t *fd_ctx = NULL;
+    int ret = -ENOMEM;
+
+    info = GF_CALLOC(sizeof(*info), 1, gf_afr_mt_lk_heal_info_t);
+    if (!info) {
+        goto cleanup;
+    }
+    INIT_LIST_HEAD(&info->pos);
+    info->fd = fd_ref(local->fd);
+    info->cmd = local->cont.lk.cmd;
+    info->pid = frame->root->pid;
+    info->flock = local->cont.lk.user_flock;
+    info->xdata_req = dict_copy_with_ref(local->xdata_req, NULL);
+    if (!info->xdata_req) {
+        goto cleanup;
+    }
+    info->lk_owner = frame->root->lk_owner;
+    info->locked_nodes = GF_MALLOC(
+        sizeof(*info->locked_nodes) * priv->child_count, gf_afr_mt_char);
+    if (!info->locked_nodes) {
+        goto cleanup;
+    }
+    memcpy(info->locked_nodes, local->cont.lk.locked_nodes,
+           sizeof(*info->locked_nodes) * priv->child_count);
+    info->child_up_event_gen = GF_CALLOC(sizeof(*info->child_up_event_gen),
+                                         priv->child_count, gf_afr_mt_int32_t);
+    if (!info->child_up_event_gen) {
+        goto cleanup;
+    }
+    info->child_down_event_gen = GF_CALLOC(sizeof(*info->child_down_event_gen),
+                                           priv->child_count,
+                                           gf_afr_mt_int32_t);
+    if (!info->child_down_event_gen) {
+        goto cleanup;
+    }
+
+    LOCK(&local->fd->lock);
+    {
+        fd_ctx = __afr_fd_ctx_get(local->fd, this);
+        if (fd_ctx)
+            fd_ctx->lk_heal_info = info;
+    }
+    UNLOCK(&local->fd->lock);
+    if (!fd_ctx) {
+        goto cleanup;
+    }
+
+    LOCK(&priv->lock);
+    {
+        list_add_tail(&info->pos, &priv->saved_locks);
+    }
+    UNLOCK(&priv->lock);
+
+    return 0;
+cleanup:
+    gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_LK_HEAL_DOM,
+           "%s: Failed to add lock to healq",
+           uuid_utoa(local->fd->inode->gfid));
+    if (info) {
+        afr_lk_heal_info_cleanup(info);
+        if (fd_ctx) {
+            LOCK(&local->fd->lock);
+            {
+                fd_ctx->lk_heal_info = NULL;
+            }
+            UNLOCK(&local->fd->lock);
         }
-        UNLOCK (&inode->lock);
+    }
+    return ret;
+}
+
+static int
+afr_remove_lock_from_saved_locks(afr_local_t *local, xlator_t *this)
+{
+    afr_private_t *priv = this->private;
+    struct gf_flock flock = local->cont.lk.user_flock;
+    afr_lk_heal_info_t *info = NULL;
+    afr_fd_ctx_t *fd_ctx = NULL;
+    int ret = -EINVAL;
+
+    fd_ctx = afr_fd_ctx_get(local->fd, this);
+    if (!fd_ctx || !fd_ctx->lk_heal_info) {
+        goto out;
+    }
+
+    info = fd_ctx->lk_heal_info;
+    if ((info->flock.l_start != flock.l_start) ||
+        (info->flock.l_whence != flock.l_whence) ||
+        (info->flock.l_len != flock.l_len)) {
+        /*TODO: Compare lkowners too.*/
+        goto out;
+    }
+
+    LOCK(&priv->lock);
+    {
+        list_del(&fd_ctx->lk_heal_info->pos);
+    }
+    UNLOCK(&priv->lock);
+
+    afr_lk_heal_info_cleanup(info);
+    fd_ctx->lk_heal_info = NULL;
+    ret = 0;
 out:
-        return;
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_LK_HEAL_DOM,
+               "%s: Failed to remove lock from healq",
+               uuid_utoa(local->fd->inode->gfid));
+    return ret;
 }
 
+int
+afr_lock_heal_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+                  dict_t *xdata)
+{
+    afr_local_t *local = frame->local;
+    int i = (long)cookie;
+
+    local->replies[i].valid = 1;
+    local->replies[i].op_ret = op_ret;
+    local->replies[i].op_errno = op_errno;
+    if (op_ret != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, AFR_MSG_LK_HEAL_DOM,
+               "Failed to heal lock on child %d for %s", i,
+               uuid_utoa(local->fd->inode->gfid));
+    }
+    syncbarrier_wake(&local->barrier);
+    return 0;
+}
 
-uint64_t
-afr_is_opendir_done (xlator_t *this, inode_t *inode)
+int
+afr_getlk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
 {
-        int ret = 0;
+    afr_local_t *local = frame->local;
+    int i = (long)cookie;
+
+    local->replies[i].valid = 1;
+    local->replies[i].op_ret = op_ret;
+    local->replies[i].op_errno = op_errno;
+    if (op_ret != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, AFR_MSG_LK_HEAL_DOM,
+               "Failed getlk for %s", uuid_utoa(local->fd->inode->gfid));
+    } else {
+        local->cont.lk.getlk_rsp[i] = *lock;
+    }
+
+    syncbarrier_wake(&local->barrier);
+    return 0;
+}
 
-        uint64_t ctx          = 0;
-        uint64_t opendir_done = 0;
+static gf_boolean_t
+afr_does_lk_owner_match(call_frame_t *frame, afr_private_t *priv,
+                        afr_lk_heal_info_t *info)
+{
+    int i = 0;
+    afr_local_t *local = frame->local;
+    struct gf_flock flock = {
+        0,
+    };
+    gf_boolean_t ret = _gf_true;
+    char *wind_on = alloca0(priv->child_count);
+    unsigned char *success_replies = alloca0(priv->child_count);
+    local->cont.lk.getlk_rsp = GF_CALLOC(sizeof(*local->cont.lk.getlk_rsp),
+                                         priv->child_count, gf_afr_mt_gf_lock);
+
+    flock = info->flock;
+    for (i = 0; i < priv->child_count; i++) {
+        if (info->locked_nodes[i])
+            wind_on[i] = 1;
+    }
+
+    AFR_ONLIST(wind_on, frame, afr_getlk_cbk, lk, info->fd, F_GETLK, &flock,
+               info->xdata_req);
+
+    afr_fill_success_replies(local, priv, success_replies);
+    if (AFR_COUNT(success_replies, priv->child_count) == 0) {
+        ret = _gf_false;
+        goto out;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid || local->replies[i].op_ret != 0)
+            continue;
+        if (local->cont.lk.getlk_rsp[i].l_type == F_UNLCK)
+            continue;
+        /*TODO: Do we really need to compare lkowner if F_UNLCK is true?*/
+        if (!is_same_lkowner(&local->cont.lk.getlk_rsp[i].l_owner,
+                             &info->lk_owner)) {
+            ret = _gf_false;
+            break;
+        }
+    }
+out:
+    afr_local_replies_wipe(local, priv);
+    GF_FREE(local->cont.lk.getlk_rsp);
+    local->cont.lk.getlk_rsp = NULL;
+    return ret;
+}
 
-        VALIDATE_OR_GOTO (inode, out);
+static void
+afr_mark_fd_bad(fd_t *fd, xlator_t *this)
+{
+    afr_fd_ctx_t *fd_ctx = NULL;
 
-        LOCK (&inode->lock);
-        {
-                ret = __inode_ctx_get (inode, this, &ctx);
+    if (!fd)
+        return;
+    LOCK(&fd->lock);
+    {
+        fd_ctx = __afr_fd_ctx_get(fd, this);
+        if (fd_ctx) {
+            fd_ctx->is_fd_bad = _gf_true;
+            fd_ctx->lk_heal_info = NULL;
+        }
+    }
+    UNLOCK(&fd->lock);
+}
 
-                if (ret < 0)
-                        goto unlock;
+static void
+afr_add_lock_to_lkhealq(afr_private_t *priv, afr_lk_heal_info_t *info)
+{
+    LOCK(&priv->lock);
+    {
+        list_del(&info->pos);
+        list_add_tail(&info->pos, &priv->lk_healq);
+    }
+    UNLOCK(&priv->lock);
+}
 
-                opendir_done = ctx & AFR_ICTX_OPENDIR_DONE_MASK;
+static void
+afr_lock_heal_do(call_frame_t *frame, afr_private_t *priv,
+                 afr_lk_heal_info_t *info)
+{
+    int i = 0;
+    int op_errno = 0;
+    int32_t *current_event_gen = NULL;
+    afr_local_t *local = frame->local;
+    xlator_t *this = frame->this;
+    char *wind_on = alloca0(priv->child_count);
+    gf_boolean_t retry = _gf_true;
+
+    frame->root->pid = info->pid;
+    lk_owner_copy(&frame->root->lk_owner, &info->lk_owner);
+
+    op_errno = -afr_dom_lock_acquire(frame);
+    if ((op_errno != 0)) {
+        goto release;
+    }
+
+    if (!afr_does_lk_owner_match(frame, priv, info)) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_LK_HEAL_DOM,
+               "Ignoring lock heal for %s since lk-onwers mismatch. "
+               "Lock possibly pre-empted by another client.",
+               uuid_utoa(info->fd->inode->gfid));
+        goto release;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (info->locked_nodes[i])
+            continue;
+        wind_on[i] = 1;
+    }
+
+    current_event_gen = alloca(priv->child_count);
+    memcpy(current_event_gen, info->child_up_event_gen,
+           priv->child_count * sizeof *current_event_gen);
+    AFR_ONLIST(wind_on, frame, afr_lock_heal_cbk, lk, info->fd, info->cmd,
+               &info->flock, info->xdata_req);
+
+    LOCK(&priv->lock);
+    {
+        for (i = 0; i < priv->child_count; i++) {
+            if (!wind_on[i])
+                continue;
+            if ((!local->replies[i].valid) || (local->replies[i].op_ret != 0)) {
+                continue;
+            }
+
+            if ((current_event_gen[i] == info->child_up_event_gen[i]) &&
+                (current_event_gen[i] > info->child_down_event_gen[i])) {
+                info->locked_nodes[i] = 1;
+                retry = _gf_false;
+                list_del_init(&info->pos);
+                list_add_tail(&info->pos, &priv->saved_locks);
+            } else {
+                /*We received subsequent child up/down events while heal was in
+                 * progress; don't mark child as healed. Attempt again on the
+                 * new child up*/
+                gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_LK_HEAL_DOM,
+                       "Event gen mismatch: skipped healing lock on child %d "
+                       "for %s.",
+                       i, uuid_utoa(info->fd->inode->gfid));
+            }
         }
-unlock:
-        UNLOCK (&inode->lock);
+    }
+    UNLOCK(&priv->lock);
+
+release:
+    afr_dom_lock_release(frame);
+    if (retry)
+        afr_add_lock_to_lkhealq(priv, info);
+    return;
+}
 
-out:
-        return opendir_done;
+static int
+afr_lock_heal_done(int ret, call_frame_t *frame, void *opaque)
+{
+    STACK_DESTROY(frame->root);
+    return 0;
 }
 
+static int
+afr_lock_heal(void *opaque)
+{
+    call_frame_t *frame = (call_frame_t *)opaque;
+    call_frame_t *iter_frame = NULL;
+    xlator_t *this = frame->this;
+    afr_private_t *priv = this->private;
+    afr_lk_heal_info_t *info = NULL;
+    afr_lk_heal_info_t *tmp = NULL;
+    struct list_head healq = {
+        0,
+    };
+    int ret = 0;
+
+    iter_frame = afr_copy_frame(frame);
+    if (!iter_frame) {
+        return ENOMEM;
+    }
+
+    INIT_LIST_HEAD(&healq);
+    LOCK(&priv->lock);
+    {
+        list_splice_init(&priv->lk_healq, &healq);
+    }
+    UNLOCK(&priv->lock);
+
+    list_for_each_entry_safe(info, tmp, &healq, pos)
+    {
+        GF_ASSERT((AFR_COUNT(info->locked_nodes, priv->child_count) <
+                   priv->child_count));
+        ((afr_local_t *)(iter_frame->local))->fd = fd_ref(info->fd);
+        afr_lock_heal_do(iter_frame, priv, info);
+        AFR_STACK_RESET(iter_frame);
+        if (iter_frame->local == NULL) {
+            ret = ENOTCONN;
+            gf_msg(frame->this->name, GF_LOG_ERROR, ENOTCONN,
+                   AFR_MSG_LK_HEAL_DOM,
+                   "Aborting processing of lk_healq."
+                   "Healing will be reattempted on next child up for locks "
+                   "that are still in quorum.");
+            LOCK(&priv->lock);
+            {
+                list_add_tail(&healq, &priv->lk_healq);
+            }
+            UNLOCK(&priv->lock);
+            break;
+        }
+    }
+
+    AFR_STACK_DESTROY(iter_frame);
+    return ret;
+}
 
-void
-afr_set_opendir_done (xlator_t *this, inode_t *inode)
+static int
+__afr_lock_heal_synctask(xlator_t *this, afr_private_t *priv, int child)
 {
-        uint64_t ctx = 0;
-        int      ret = 0;
+    int ret = 0;
+    call_frame_t *frame = NULL;
+    afr_lk_heal_info_t *info = NULL;
+    afr_lk_heal_info_t *tmp = NULL;
 
-        VALIDATE_OR_GOTO (inode, out);
+    if (priv->shd.iamshd)
+        return 0;
 
-        LOCK (&inode->lock);
-        {
-                ret = __inode_ctx_get (inode, this, &ctx);
+    list_for_each_entry_safe(info, tmp, &priv->saved_locks, pos)
+    {
+        info->child_up_event_gen[child] = priv->event_generation;
+        list_del_init(&info->pos);
+        list_add_tail(&info->pos, &priv->lk_healq);
+    }
 
-                if (ret < 0) {
-                        ctx = 0;
-                }
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame)
+        return -1;
 
-                ctx = (~AFR_ICTX_OPENDIR_DONE_MASK & ctx)
-                        | (0xFFFFFFFFFFFFFFFFULL & AFR_ICTX_OPENDIR_DONE_MASK);
+    ret = synctask_new(this->ctx->env, afr_lock_heal, afr_lock_heal_done, frame,
+                       frame);
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_LK_HEAL_DOM,
+               "Failed to launch lock heal synctask");
 
-                __inode_ctx_put (inode, this, ctx);
-        }
-        UNLOCK (&inode->lock);
-out:
-        return;
+    return ret;
 }
 
-
-uint64_t
-afr_read_child (xlator_t *this, inode_t *inode)
+static int
+__afr_mark_pending_lk_heal(xlator_t *this, afr_private_t *priv, int child)
 {
-        int ret = 0;
-
-        uint64_t ctx         = 0;
-        uint64_t read_child  = 0;
+    afr_lk_heal_info_t *info = NULL;
+    afr_lk_heal_info_t *tmp = NULL;
 
-        VALIDATE_OR_GOTO (inode, out);
+    if (priv->shd.iamshd)
+        return 0;
+    list_for_each_entry_safe(info, tmp, &priv->saved_locks, pos)
+    {
+        info->child_down_event_gen[child] = priv->event_generation;
+        if (info->locked_nodes[child] == 1)
+            info->locked_nodes[child] = 0;
+        if (!afr_has_quorum(info->locked_nodes, this, NULL)) {
+            /* Since the lock was lost on quorum no. of nodes, we should
+             * not attempt to heal it anymore. Some other client could have
+             * acquired the lock, modified data and released it and this
+             * client wouldn't know about it if we heal it.*/
+            afr_mark_fd_bad(info->fd, this);
+            list_del(&info->pos);
+            afr_lk_heal_info_cleanup(info);
+            /* We're not winding an unlock on the node where the lock is still
+             * present because when fencing logic switches over to the new
+             * client (since we marked the fd bad), it should preempt any
+             * existing lock. */
+        }
+    }
+    return 0;
+}
 
-        LOCK (&inode->lock);
-        {
-                ret = __inode_ctx_get (inode, this, &ctx);
+gf_boolean_t
+afr_is_consistent_io_possible(afr_local_t *local, afr_private_t *priv,
+                              int32_t *op_errno)
+{
+    if (priv->consistent_io && local->call_count != priv->child_count) {
+        gf_msg(THIS->name, GF_LOG_INFO, 0, AFR_MSG_SUBVOLS_DOWN,
+               "All subvolumes are not up");
+        if (op_errno)
+            *op_errno = ENOTCONN;
+        return _gf_false;
+    }
+    return _gf_true;
+}
 
-                if (ret < 0)
-                        goto unlock;
+gf_boolean_t
+afr_is_lock_mode_mandatory(dict_t *xdata)
+{
+    int ret = 0;
+    uint32_t lk_mode = GF_LK_ADVISORY;
 
-                read_child = ctx & AFR_ICTX_READ_CHILD_MASK;
-        }
-unlock:
-        UNLOCK (&inode->lock);
+    ret = dict_get_uint32(xdata, GF_LOCK_MODE, &lk_mode);
+    if (!ret && lk_mode == GF_LK_MANDATORY)
+        return _gf_true;
 
-out:
-        return read_child;
+    return _gf_false;
 }
 
+call_frame_t *
+afr_copy_frame(call_frame_t *base)
+{
+    afr_local_t *local = NULL;
+    call_frame_t *frame = NULL;
+    int op_errno = 0;
+
+    frame = copy_frame(base);
+    if (!frame)
+        return NULL;
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local) {
+        AFR_STACK_DESTROY(frame);
+        return NULL;
+    }
+
+    return frame;
+}
 
-void
-afr_set_read_child (xlator_t *this, inode_t *inode, int32_t read_child)
+/* Check if an entry or inode could be undergoing a transaction. */
+gf_boolean_t
+afr_is_possibly_under_txn(afr_transaction_type type, afr_local_t *local,
+                          xlator_t *this)
 {
-        uint64_t ctx = 0;
-        int      ret = 0;
+    int i = 0;
+    int tmp = 0;
+    afr_private_t *priv = NULL;
+    GF_UNUSED char *key = NULL;
+    int keylen = 0;
+
+    priv = this->private;
+
+    if (type == AFR_ENTRY_TRANSACTION) {
+        key = GLUSTERFS_PARENT_ENTRYLK;
+        keylen = SLEN(GLUSTERFS_PARENT_ENTRYLK);
+    } else if (type == AFR_DATA_TRANSACTION) {
+        /*FIXME: Use GLUSTERFS_INODELK_DOM_COUNT etc. once
+         * pl_inodelk_xattr_fill supports separate keys for different
+         * domains.*/
+        key = GLUSTERFS_INODELK_COUNT;
+        keylen = SLEN(GLUSTERFS_INODELK_COUNT);
+    }
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].xdata)
+            continue;
+        if (dict_get_int32n(local->replies[i].xdata, key, keylen, &tmp) == 0)
+            if (tmp)
+                return _gf_true;
+    }
+
+    return _gf_false;
+}
 
-        VALIDATE_OR_GOTO (inode, out);
+static void
+afr_inode_ctx_destroy(afr_inode_ctx_t *ctx)
+{
+    int i = 0;
 
-        LOCK (&inode->lock);
-        {
-                ret = __inode_ctx_get (inode, this, &ctx);
+    if (!ctx)
+        return;
 
-                if (ret < 0) {
-                        ctx = 0;
-                }
+    for (i = 0; i < AFR_NUM_CHANGE_LOGS; i++) {
+        GF_FREE(ctx->pre_op_done[i]);
+    }
 
-                ctx = (~AFR_ICTX_READ_CHILD_MASK & ctx)
-                        | (AFR_ICTX_READ_CHILD_MASK & read_child);
+    GF_FREE(ctx);
+}
 
-                __inode_ctx_put (inode, this, ctx);
+int
+__afr_inode_ctx_get(xlator_t *this, inode_t *inode, afr_inode_ctx_t **ctx)
+{
+    uint64_t ctx_int = 0;
+    int ret = -1;
+    int i = -1;
+    int num_locks = -1;
+    afr_inode_ctx_t *ictx = NULL;
+    afr_lock_t *lock = NULL;
+    afr_private_t *priv = this->private;
+
+    ret = __inode_ctx_get(inode, this, &ctx_int);
+    if (ret == 0) {
+        *ctx = (afr_inode_ctx_t *)(uintptr_t)ctx_int;
+        return 0;
+    }
+
+    ictx = GF_CALLOC(1, sizeof(afr_inode_ctx_t), gf_afr_mt_inode_ctx_t);
+    if (!ictx)
+        goto out;
+
+    for (i = 0; i < AFR_NUM_CHANGE_LOGS; i++) {
+        ictx->pre_op_done[i] = GF_CALLOC(sizeof *ictx->pre_op_done[i],
+                                         priv->child_count, gf_afr_mt_int32_t);
+        if (!ictx->pre_op_done[i]) {
+            ret = -ENOMEM;
+            goto out;
         }
-        UNLOCK (&inode->lock);
-
+    }
+
+    num_locks = sizeof(ictx->lock) / sizeof(afr_lock_t);
+    for (i = 0; i < num_locks; i++) {
+        lock = &ictx->lock[i];
+        INIT_LIST_HEAD(&lock->post_op);
+        INIT_LIST_HEAD(&lock->frozen);
+        INIT_LIST_HEAD(&lock->waiting);
+        INIT_LIST_HEAD(&lock->owners);
+    }
+
+    ctx_int = (uint64_t)(uintptr_t)ictx;
+    ret = __inode_ctx_set(inode, this, &ctx_int);
+    if (ret) {
+        goto out;
+    }
+
+    ictx->spb_choice = -1;
+    ictx->read_subvol = 0;
+    ictx->write_subvol = 0;
+    ictx->lock_count = 0;
+    ret = 0;
+    *ctx = ictx;
 out:
-        return;
+    if (ret) {
+        afr_inode_ctx_destroy(ictx);
+    }
+    return ret;
 }
 
-
-/**
- * afr_local_cleanup - cleanup everything in frame->local
+/*
+ * INODE CTX 64-bit VALUE FORMAT FOR SMALL (<= 16) SUBVOL COUNTS:
+ *
+ * |<----------   64bit   ------------>|
+ *  63           32 31    16 15       0
+ * |   EVENT_GEN   |  DATA  | METADATA |
+ *
+ *
+ *  METADATA (bit-0 .. bit-15): bitmap representing subvolumes from which
+ *                              metadata can be attempted to be read.
+ *
+ *                              bit-0 => priv->subvolumes[0]
+ *                              bit-1 => priv->subvolumes[1]
+ *                              ... etc. till bit-15
+ *
+ *  DATA (bit-16 .. bit-31): bitmap representing subvolumes from which data
+ *                           can be attempted to be read.
+ *
+ *                           bit-16 => priv->subvolumes[0]
+ *                           bit-17 => priv->subvolumes[1]
+ *                           ... etc. till bit-31
+ *
+ *  EVENT_GEN (bit-32 .. bit-63): event generation (i.e priv->event_generation)
+ *                                when DATA and METADATA was last updated.
+ *
+ *                                If EVENT_GEN is < priv->event_generation,
+ *                                or is 0, it means afr_inode_refresh() needs
+ *                                to be called to recalculate the bitmaps.
  */
 
-void
-afr_local_sh_cleanup (afr_local_t *local, xlator_t *this)
+int
+__afr_set_in_flight_sb_status(xlator_t *this, afr_local_t *local,
+                              inode_t *inode)
 {
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        int              i = 0;
+    int i = 0;
+    int txn_type = 0;
+    int count = 0;
+    int index = -1;
+    uint16_t datamap_old = 0;
+    uint16_t metadatamap_old = 0;
+    uint16_t datamap = 0;
+    uint16_t metadatamap = 0;
+    uint16_t tmp_map = 0;
+    uint16_t mask = 0;
+    uint32_t event = 0;
+    uint64_t val = 0;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+    txn_type = local->transaction.type;
+
+    if (txn_type == AFR_DATA_TRANSACTION)
+        val = local->inode_ctx->write_subvol;
+    else
+        val = local->inode_ctx->read_subvol;
+
+    metadatamap_old = metadatamap = (val & 0x000000000000ffff);
+    datamap_old = datamap = (val & 0x00000000ffff0000) >> 16;
+    event = (val & 0xffffffff00000000) >> 32;
+
+    if (txn_type == AFR_DATA_TRANSACTION)
+        tmp_map = datamap;
+    else if (txn_type == AFR_METADATA_TRANSACTION)
+        tmp_map = metadatamap;
+
+    count = gf_bits_count(tmp_map);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->transaction.failed_subvols[i])
+            continue;
+
+        mask = 1 << i;
+        if (txn_type == AFR_METADATA_TRANSACTION)
+            metadatamap &= ~mask;
+        else if (txn_type == AFR_DATA_TRANSACTION)
+            datamap &= ~mask;
+    }
+
+    switch (txn_type) {
+        case AFR_METADATA_TRANSACTION:
+            if ((metadatamap_old != 0) && (metadatamap == 0) && (count == 1)) {
+                index = gf_bits_index(tmp_map);
+                local->transaction.in_flight_sb_errno = local->replies[index]
+                                                            .op_errno;
+                local->transaction.in_flight_sb = _gf_true;
+                metadatamap |= (1 << index);
+            }
+            if (metadatamap_old != metadatamap) {
+                __afr_inode_need_refresh_set(inode, this);
+            }
+            break;
+
+        case AFR_DATA_TRANSACTION:
+            if ((datamap_old != 0) && (datamap == 0) && (count == 1)) {
+                index = gf_bits_index(tmp_map);
+                local->transaction.in_flight_sb_errno = local->replies[index]
+                                                            .op_errno;
+                local->transaction.in_flight_sb = _gf_true;
+                datamap |= (1 << index);
+            }
+            if (datamap_old != datamap)
+                __afr_inode_need_refresh_set(inode, this);
+            break;
 
+        default:
+            break;
+    }
 
-        sh = &local->self_heal;
-        priv = this->private;
+    val = ((uint64_t)metadatamap) | (((uint64_t)datamap) << 16) |
+          (((uint64_t)event) << 32);
 
-        if (sh->buf)
-                GF_FREE (sh->buf);
+    if (txn_type == AFR_DATA_TRANSACTION)
+        local->inode_ctx->write_subvol = val;
+    local->inode_ctx->read_subvol = val;
 
-        if (sh->xattr) {
-                for (i = 0; i < priv->child_count; i++) {
-                        if (sh->xattr[i]) {
-                                dict_unref (sh->xattr[i]);
-                                sh->xattr[i] = NULL;
-                        }
-                }
-                GF_FREE (sh->xattr);
-        }
-
-        if (sh->child_errno)
-                GF_FREE (sh->child_errno);
+    return 0;
+}
 
-        if (sh->pending_matrix) {
-                for (i = 0; i < priv->child_count; i++) {
-                        GF_FREE (sh->pending_matrix[i]);
-                }
-                GF_FREE (sh->pending_matrix);
+gf_boolean_t
+afr_is_symmetric_error(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int op_errno = 0;
+    int i_errno = 0;
+    gf_boolean_t matching_errors = _gf_true;
+    int i = 0;
+
+    priv = this->private;
+    local = frame->local;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid)
+            continue;
+        if (local->replies[i].op_ret != -1) {
+            /* Operation succeeded on at least one subvol,
+               so it is not a failed-everywhere situation.
+            */
+            matching_errors = _gf_false;
+            break;
+        }
+        i_errno = local->replies[i].op_errno;
+
+        if (i_errno == ENOTCONN) {
+            /* ENOTCONN is not a symmetric error. We do not
+               know if the operation was performed on the
+               backend or not.
+            */
+            matching_errors = _gf_false;
+            break;
         }
 
-        if (sh->delta_matrix) {
-                for (i = 0; i < priv->child_count; i++) {
-                        GF_FREE (sh->delta_matrix[i]);
-                }
-                GF_FREE (sh->delta_matrix);
+        if (!op_errno) {
+            op_errno = i_errno;
+        } else if (op_errno != i_errno) {
+            /* Mismatching op_errno's */
+            matching_errors = _gf_false;
+            break;
         }
+    }
 
-        if (sh->sources)
-                GF_FREE (sh->sources);
+    return matching_errors;
+}
 
-        if (sh->success)
-                GF_FREE (sh->success);
+int
+afr_set_in_flight_sb_status(xlator_t *this, call_frame_t *frame, inode_t *inode)
+{
+    int ret = -1;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
 
-        if (sh->locked_nodes)
-                GF_FREE (sh->locked_nodes);
+    priv = this->private;
+    local = frame->local;
 
-        if (sh->healing_fd && !sh->healing_fd_opened) {
-                fd_unref (sh->healing_fd);
-                sh->healing_fd = NULL;
-        }
+    /* If this transaction saw no failures, then exit. */
+    if (AFR_COUNT(local->transaction.failed_subvols, priv->child_count) == 0)
+        return 0;
 
-        if (sh->linkname)
-                GF_FREE ((char *)sh->linkname);
+    if (afr_is_symmetric_error(frame, this))
+        return 0;
 
-        loc_wipe (&sh->parent_loc);
-}
+    LOCK(&inode->lock);
+    {
+        ret = __afr_set_in_flight_sb_status(this, local, inode);
+    }
+    UNLOCK(&inode->lock);
 
+    return ret;
+}
 
-void
-afr_local_transaction_cleanup (afr_local_t *local, xlator_t *this)
+int
+__afr_inode_read_subvol_get_small(inode_t *inode, xlator_t *this,
+                                  unsigned char *data, unsigned char *metadata,
+                                  int *event_p)
 {
-        int             i = 0;
-        afr_private_t * priv = NULL;
+    afr_private_t *priv = NULL;
+    int ret = -1;
+    uint16_t datamap = 0;
+    uint16_t metadatamap = 0;
+    uint32_t event = 0;
+    uint64_t val = 0;
+    int i = 0;
+    afr_inode_ctx_t *ctx = NULL;
+
+    priv = this->private;
+
+    ret = __afr_inode_ctx_get(this, inode, &ctx);
+    if (ret < 0)
+        return ret;
 
-        priv = this->private;
+    val = ctx->read_subvol;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->pending && local->pending[i])
-                        GF_FREE (local->pending[i]);
-        }
+    metadatamap = (val & 0x000000000000ffff);
+    datamap = (val & 0x00000000ffff0000) >> 16;
+    event = (val & 0xffffffff00000000) >> 32;
 
-        GF_FREE (local->pending);
+    for (i = 0; i < priv->child_count; i++) {
+        if (metadata)
+            metadata[i] = (metadatamap >> i) & 1;
+        if (data)
+            data[i] = (datamap >> i) & 1;
+    }
 
-        if (local->internal_lock.locked_nodes)
-                GF_FREE (local->internal_lock.locked_nodes);
+    if (event_p)
+        *event_p = event;
+    return ret;
+}
 
-        if (local->internal_lock.inode_locked_nodes)
-                GF_FREE (local->internal_lock.inode_locked_nodes);
+int
+__afr_inode_read_subvol_set_small(inode_t *inode, xlator_t *this,
+                                  unsigned char *data, unsigned char *metadata,
+                                  int event)
+{
+    afr_private_t *priv = NULL;
+    uint16_t datamap = 0;
+    uint16_t metadatamap = 0;
+    uint64_t val = 0;
+    int i = 0;
+    int ret = -1;
+    afr_inode_ctx_t *ctx = NULL;
 
-        if (local->internal_lock.entry_locked_nodes)
-                GF_FREE (local->internal_lock.entry_locked_nodes);
+    priv = this->private;
 
-        if (local->internal_lock.lower_locked_nodes)
-                GF_FREE (local->internal_lock.lower_locked_nodes);
+    ret = __afr_inode_ctx_get(this, inode, &ctx);
+    if (ret)
+        goto out;
 
+    for (i = 0; i < priv->child_count; i++) {
+        if (data[i])
+            datamap |= (1 << i);
+        if (metadata[i])
+            metadatamap |= (1 << i);
+    }
 
-        GF_FREE (local->transaction.child_errno);
-        GF_FREE (local->child_errno);
+    val = ((uint64_t)metadatamap) | (((uint64_t)datamap) << 16) |
+          (((uint64_t)event) << 32);
 
-        GF_FREE (local->transaction.basename);
-        GF_FREE (local->transaction.new_basename);
+    ctx->read_subvol = val;
 
-        loc_wipe (&local->transaction.parent_loc);
-        loc_wipe (&local->transaction.new_parent_loc);
+    ret = 0;
+out:
+    return ret;
 }
 
-
-void
-afr_local_cleanup (afr_local_t *local, xlator_t *this)
+int
+__afr_inode_read_subvol_get(inode_t *inode, xlator_t *this, unsigned char *data,
+                            unsigned char *metadata, int *event_p)
 {
-        int i;
-        afr_private_t * priv = NULL;
+    afr_private_t *priv = NULL;
+    int ret = -1;
 
-        if (!local)
-                return;
+    priv = this->private;
 
-        afr_local_sh_cleanup (local, this);
+    if (priv->child_count <= 16)
+        ret = __afr_inode_read_subvol_get_small(inode, this, data, metadata,
+                                                event_p);
+    else
+        /* TBD: allocate structure with array and read from it */
+        ret = -1;
 
-        afr_local_transaction_cleanup (local, this);
+    return ret;
+}
 
-        priv = this->private;
+int
+__afr_inode_split_brain_choice_get(inode_t *inode, xlator_t *this,
+                                   int *spb_choice)
+{
+    afr_inode_ctx_t *ctx = NULL;
+    int ret = -1;
 
-        loc_wipe (&local->loc);
-        loc_wipe (&local->newloc);
+    ret = __afr_inode_ctx_get(this, inode, &ctx);
+    if (ret < 0)
+        return ret;
 
-        if (local->fd)
-                fd_unref (local->fd);
-
-        if (local->xattr_req)
-                dict_unref (local->xattr_req);
-
-        GF_FREE (local->child_up);
-
-        { /* lookup */
-                if (local->cont.lookup.xattrs) {
-                        for (i = 0; i < priv->child_count; i++) {
-                                if (local->cont.lookup.xattrs[i]) {
-                                        dict_unref (local->cont.lookup.xattrs[i]);
-                                        local->cont.lookup.xattrs[i] = NULL;
-                                }
-                        }
-                        GF_FREE (local->cont.lookup.xattrs);
-                        local->cont.lookup.xattrs = NULL;
-                }
+    *spb_choice = ctx->spb_choice;
+    return 0;
+}
 
-                if (local->cont.lookup.xattr) {
-                        dict_unref (local->cont.lookup.xattr);
-                }
+int
+__afr_inode_read_subvol_set(inode_t *inode, xlator_t *this, unsigned char *data,
+                            unsigned char *metadata, int event)
+{
+    afr_private_t *priv = NULL;
+    int ret = -1;
 
-                if (local->cont.lookup.inode) {
-                        inode_unref (local->cont.lookup.inode);
-                }
-        }
+    priv = this->private;
 
-        { /* getxattr */
-                if (local->cont.getxattr.name)
-                        GF_FREE (local->cont.getxattr.name);
-        }
+    if (priv->child_count <= 16)
+        ret = __afr_inode_read_subvol_set_small(inode, this, data, metadata,
+                                                event);
+    else
+        ret = -1;
 
-        { /* lk */
-                if (local->cont.lk.locked_nodes)
-                        GF_FREE (local->cont.lk.locked_nodes);
-        }
+    return ret;
+}
 
-        { /* create */
-                if (local->cont.create.fd)
-                        fd_unref (local->cont.create.fd);
-                if (local->cont.create.params)
-                        dict_unref (local->cont.create.params);
-        }
+int
+__afr_inode_split_brain_choice_set(inode_t *inode, xlator_t *this,
+                                   int spb_choice)
+{
+    afr_inode_ctx_t *ctx = NULL;
+    int ret = -1;
 
-        { /* mknod */
-                if (local->cont.mknod.params)
-                        dict_unref (local->cont.mknod.params);
-        }
+    ret = __afr_inode_ctx_get(this, inode, &ctx);
+    if (ret)
+        goto out;
 
-        { /* mkdir */
-                if (local->cont.mkdir.params)
-                        dict_unref (local->cont.mkdir.params);
-        }
+    ctx->spb_choice = spb_choice;
 
-        { /* symlink */
-                if (local->cont.symlink.params)
-                        dict_unref (local->cont.symlink.params);
-        }
+    ret = 0;
+out:
+    return ret;
+}
 
-        { /* writev */
-                GF_FREE (local->cont.writev.vector);
-        }
+int
+afr_inode_read_subvol_get(inode_t *inode, xlator_t *this, unsigned char *data,
+                          unsigned char *metadata, int *event_p)
+{
+    int ret = -1;
 
-        { /* setxattr */
-                if (local->cont.setxattr.dict)
-                        dict_unref (local->cont.setxattr.dict);
-        }
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
 
-        { /* removexattr */
-                GF_FREE (local->cont.removexattr.name);
-        }
+    LOCK(&inode->lock);
+    {
+        ret = __afr_inode_read_subvol_get(inode, this, data, metadata, event_p);
+    }
+    UNLOCK(&inode->lock);
+out:
+    return ret;
+}
 
-        { /* symlink */
-                GF_FREE (local->cont.symlink.linkpath);
+int
+afr_inode_get_readable(call_frame_t *frame, inode_t *inode, xlator_t *this,
+                       unsigned char *readable, int *event_p, int type)
+{
+    afr_private_t *priv = this->private;
+    afr_local_t *local = frame->local;
+    unsigned char *data = alloca0(priv->child_count);
+    unsigned char *metadata = alloca0(priv->child_count);
+    int data_count = 0;
+    int metadata_count = 0;
+    int event_generation = 0;
+    int ret = 0;
+
+    ret = afr_inode_read_subvol_get(inode, this, data, metadata,
+                                    &event_generation);
+    if (ret == -1)
+        return -EIO;
+
+    data_count = AFR_COUNT(data, priv->child_count);
+    metadata_count = AFR_COUNT(metadata, priv->child_count);
+
+    if (inode->ia_type == IA_IFDIR) {
+        /* For directories, allow even if it is in data split-brain. */
+        if (type == AFR_METADATA_TRANSACTION || local->op == GF_FOP_STAT ||
+            local->op == GF_FOP_FSTAT) {
+            if (!metadata_count)
+                return -EIO;
         }
-
-        { /* opendir */
-                if (local->cont.opendir.checksum)
-                        GF_FREE (local->cont.opendir.checksum);
+    } else {
+        /* For files, abort in case of data/metadata split-brain. */
+        if (!data_count || !metadata_count) {
+            return -EIO;
         }
+    }
+
+    if (type == AFR_METADATA_TRANSACTION && readable)
+        memcpy(readable, metadata, priv->child_count * sizeof *metadata);
+    if (type == AFR_DATA_TRANSACTION && readable) {
+        if (!data_count)
+            memcpy(readable, local->child_up,
+                   priv->child_count * sizeof *readable);
+        else
+            memcpy(readable, data, priv->child_count * sizeof *data);
+    }
+    if (event_p)
+        *event_p = event_generation;
+    return 0;
 }
 
+static int
+afr_inode_split_brain_choice_get(inode_t *inode, xlator_t *this,
+                                 int *spb_choice)
+{
+    int ret = -1;
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
+
+    LOCK(&inode->lock);
+    {
+        ret = __afr_inode_split_brain_choice_get(inode, this, spb_choice);
+    }
+    UNLOCK(&inode->lock);
+out:
+    return ret;
+}
 
+/*
+ * frame is used to get the favourite policy. Since
+ * afr_inode_split_brain_choice_get was called with afr_open, it is possible to
+ * have a frame with out local->replies. So in that case, frame is passed as
+ * null, hence this function will handle the frame NULL case.
+ */
 int
-afr_frame_return (call_frame_t *frame)
+afr_split_brain_read_subvol_get(inode_t *inode, xlator_t *this,
+                                call_frame_t *frame, int *spb_subvol)
 {
-        afr_local_t *local = NULL;
-        int          call_count = 0;
+    int ret = -1;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        local = frame->local;
+    GF_VALIDATE_OR_GOTO("afr", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
+    GF_VALIDATE_OR_GOTO(this->name, spb_subvol, out);
 
-        LOCK (&frame->lock);
-        {
-                call_count = --local->call_count;
+    priv = this->private;
+
+    ret = afr_inode_split_brain_choice_get(inode, this, spb_subvol);
+    if (*spb_subvol < 0 && priv->fav_child_policy && frame && frame->local) {
+        local = frame->local;
+        *spb_subvol = afr_sh_get_fav_by_policy(this, local->replies, inode,
+                                               NULL);
+        if (*spb_subvol >= 0) {
+            ret = 0;
         }
-        UNLOCK (&frame->lock);
+    }
 
-        return call_count;
+out:
+    return ret;
 }
+int
+afr_inode_read_subvol_set(inode_t *inode, xlator_t *this, unsigned char *data,
+                          unsigned char *metadata, int event)
+{
+    int ret = -1;
 
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
 
-/**
- * up_children_count - return the number of children that are up
- */
+    LOCK(&inode->lock);
+    {
+        ret = __afr_inode_read_subvol_set(inode, this, data, metadata, event);
+    }
+    UNLOCK(&inode->lock);
+out:
+    return ret;
+}
 
 int
-afr_up_children_count (int child_count, unsigned char *child_up)
+afr_inode_split_brain_choice_set(inode_t *inode, xlator_t *this, int spb_choice)
 {
-        int i   = 0;
-        int ret = 0;
+    int ret = -1;
 
-        for (i = 0; i < child_count; i++)
-                if (child_up[i])
-                        ret++;
-        return ret;
-}
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
 
+    LOCK(&inode->lock);
+    {
+        ret = __afr_inode_split_brain_choice_set(inode, this, spb_choice);
+    }
+    UNLOCK(&inode->lock);
+out:
+    return ret;
+}
 
-ino64_t
-afr_itransform (ino64_t ino, int child_count, int child_index)
+/* The caller of this should perform afr_inode_refresh, if this function
+ * returns _gf_true
+ */
+gf_boolean_t
+afr_is_inode_refresh_reqd(inode_t *inode, xlator_t *this, int event_gen1,
+                          int event_gen2)
 {
-        ino64_t scaled_ino = -1;
+    gf_boolean_t need_refresh = _gf_false;
+    afr_inode_ctx_t *ctx = NULL;
+    int ret = -1;
 
-        if (ino == ((uint64_t) -1)) {
-                scaled_ino = ((uint64_t) -1);
-                goto out;
-        }
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
 
-        scaled_ino = (ino * child_count) + child_index;
+    LOCK(&inode->lock);
+    {
+        ret = __afr_inode_ctx_get(this, inode, &ctx);
+        if (ret)
+            goto unlock;
+
+        need_refresh = ctx->need_refresh;
+        /* Hoping that the caller will do inode_refresh followed by
+         * this, hence setting the need_refresh to false */
+        ctx->need_refresh = _gf_false;
+    }
+unlock:
+    UNLOCK(&inode->lock);
 
+    if (event_gen1 != event_gen2)
+        need_refresh = _gf_true;
 out:
-        return scaled_ino;
+    return need_refresh;
 }
 
-
 int
-afr_deitransform_orig (ino64_t ino, int child_count)
+__afr_inode_need_refresh_set(inode_t *inode, xlator_t *this)
 {
-        int index = -1;
+    int ret = -1;
+    afr_inode_ctx_t *ctx = NULL;
 
-        index = ino % child_count;
+    ret = __afr_inode_ctx_get(this, inode, &ctx);
+    if (ret == 0) {
+        ctx->need_refresh = _gf_true;
+    }
 
-        return index;
+    return ret;
 }
 
-
 int
-afr_deitransform (ino64_t ino, int child_count)
+afr_inode_need_refresh_set(inode_t *inode, xlator_t *this)
 {
-        return 0;
-}
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
 
+    LOCK(&inode->lock);
+    {
+        ret = __afr_inode_need_refresh_set(inode, this);
+    }
+    UNLOCK(&inode->lock);
+out:
+    return ret;
+}
 
 int
-afr_self_heal_lookup_unwind (call_frame_t *frame, xlator_t *this)
+afr_spb_choice_timeout_cancel(xlator_t *this, inode_t *inode)
 {
-        afr_local_t *local = NULL;
+    afr_inode_ctx_t *ctx = NULL;
+    int ret = -1;
 
-        local = frame->local;
+    if (!inode)
+        return ret;
 
-        if (local->govinda_gOvinda) {
-                afr_set_split_brain (this, local->cont.lookup.inode, _gf_true);
+    LOCK(&inode->lock);
+    {
+        ret = __afr_inode_ctx_get(this, inode, &ctx);
+        if (ret < 0 || !ctx) {
+            UNLOCK(&inode->lock);
+            gf_msg(this->name, GF_LOG_WARNING, 0,
+                   AFR_MSG_SPLIT_BRAIN_CHOICE_ERROR,
+                   "Failed to cancel split-brain choice timer.");
+            goto out;
         }
-
-        AFR_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
-                          local->cont.lookup.inode,
-                          &local->cont.lookup.buf,
-                          local->cont.lookup.xattr,
-                          &local->cont.lookup.postparent);
-
-        return 0;
+        ctx->spb_choice = -1;
+        if (ctx->timer) {
+            gf_timer_call_cancel(this->ctx, ctx->timer);
+            ctx->timer = NULL;
+        }
+        ret = 0;
+    }
+    UNLOCK(&inode->lock);
+out:
+    return ret;
 }
 
-
-static void
-afr_lookup_collect_xattr (afr_local_t *local, xlator_t *this,
-                          int child_index, dict_t *xattr)
+void
+afr_set_split_brain_choice_cbk(void *data)
 {
-        uint32_t        inodelk_count = 0;
-        uint32_t        entrylk_count = 0;
+    inode_t *inode = data;
+    xlator_t *this = THIS;
 
-        int ret = 0;
+    afr_spb_choice_timeout_cancel(this, inode);
+    inode_invalidate(inode);
+    inode_unref(inode);
+    return;
+}
 
-        if (afr_sh_has_metadata_pending (xattr, child_index, this)) {
-                local->self_heal.need_metadata_self_heal = _gf_true;
-                gf_log(this->name, GF_LOG_DEBUG,
-                       "metadata self-heal is pending for %s.",
-                       local->loc.path);
+int
+afr_set_split_brain_choice(int ret, call_frame_t *frame, void *opaque)
+{
+    int op_errno = ENOMEM;
+    afr_private_t *priv = NULL;
+    afr_inode_ctx_t *ctx = NULL;
+    inode_t *inode = NULL;
+    loc_t *loc = NULL;
+    xlator_t *this = NULL;
+    afr_spbc_timeout_t *data = opaque;
+    struct timespec delta = {
+        0,
+    };
+    gf_boolean_t timer_set = _gf_false;
+    gf_boolean_t timer_cancelled = _gf_false;
+    gf_boolean_t timer_reset = _gf_false;
+    int old_spb_choice = -1;
+
+    frame = data->frame;
+    loc = data->loc;
+    this = frame->this;
+    priv = this->private;
+
+    if (ret) {
+        op_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+
+    delta.tv_sec = priv->spb_choice_timeout;
+    delta.tv_nsec = 0;
+
+    if (!loc->inode) {
+        ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    if (!(data->d_spb || data->m_spb)) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_SPLIT_BRAIN_CHOICE_ERROR,
+               "Cannot set "
+               "replica.split-brain-choice on %s. File is"
+               " not in data/metadata split-brain.",
+               uuid_utoa(loc->gfid));
+        ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    /*
+     * we're ref'ing the inode before LOCK like it is done elsewhere in the
+     * code. If we ref after LOCK, coverity complains of possible deadlocks.
+     */
+    inode = inode_ref(loc->inode);
+
+    LOCK(&inode->lock);
+    {
+        ret = __afr_inode_ctx_get(this, inode, &ctx);
+        if (ret) {
+            UNLOCK(&inode->lock);
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   AFR_MSG_SPLIT_BRAIN_CHOICE_ERROR,
+                   "Failed to get inode_ctx for %s", loc->name);
+            goto post_unlock;
         }
 
-        if (afr_sh_has_entry_pending (xattr, child_index, this)) {
-                local->self_heal.need_entry_self_heal = _gf_true;
-                gf_log(this->name, GF_LOG_DEBUG,
-                       "entry self-heal is pending for %s.", local->loc.path);
+        old_spb_choice = ctx->spb_choice;
+        ctx->spb_choice = data->spb_child_index;
+
+        /* Possible changes in spb-choice :
+         *         valid to -1    : cancel timer and unref
+         *         valid to valid : cancel timer and inject new one
+         *         -1    to -1    : unref and do not do anything
+         *         -1 to valid    : inject timer
+         */
+
+        /* ctx->timer is NULL iff previous value of
+         * ctx->spb_choice is -1
+         */
+        if (ctx->timer) {
+            if (ctx->spb_choice == -1) {
+                if (!gf_timer_call_cancel(this->ctx, ctx->timer)) {
+                    ctx->timer = NULL;
+                    timer_cancelled = _gf_true;
+                }
+                /* If timer cancel failed here it means that the
+                 *  previous cbk will be executed which will set
+                 *  spb_choice to -1. So we can consider the
+                 *  'valid to -1' case to be a success
+                 *  (i.e. ret = 0) and goto unlock.
+                 */
+                goto unlock;
+            }
+            goto reset_timer;
+        } else {
+            if (ctx->spb_choice == -1)
+                goto unlock;
+            goto set_timer;
         }
 
-        if (afr_sh_has_data_pending (xattr, child_index, this)) {
-                local->self_heal.need_data_self_heal = _gf_true;
-                gf_log(this->name, GF_LOG_DEBUG,
-                       "data self-heal is pending for %s.", local->loc.path);
+    reset_timer:
+        ret = gf_timer_call_cancel(this->ctx, ctx->timer);
+        if (ret != 0) {
+            /* We need to bail out now instead of launching a new
+             * timer. Otherwise the cbk of the previous timer event
+             * will cancel the new ctx->timer.
+             */
+            ctx->spb_choice = old_spb_choice;
+            ret = -1;
+            op_errno = EAGAIN;
+            goto unlock;
         }
-
-        ret = dict_get_uint32 (xattr, GLUSTERFS_INODELK_COUNT,
-                               &inodelk_count);
-        if (ret == 0)
-                local->inodelk_count += inodelk_count;
-
-        ret = dict_get_uint32 (xattr, GLUSTERFS_ENTRYLK_COUNT,
-                               &entrylk_count);
-        if (ret == 0)
-                local->entrylk_count += entrylk_count;
+        ctx->timer = NULL;
+        timer_reset = _gf_true;
+
+    set_timer:
+        ctx->timer = gf_timer_call_after(this->ctx, delta,
+                                         afr_set_split_brain_choice_cbk, inode);
+        if (!ctx->timer) {
+            ctx->spb_choice = old_spb_choice;
+            ret = -1;
+            op_errno = ENOMEM;
+        }
+        if (!timer_reset && ctx->timer)
+            timer_set = _gf_true;
+        if (timer_reset && !ctx->timer)
+            timer_cancelled = _gf_true;
+    }
+unlock:
+    UNLOCK(&inode->lock);
+post_unlock:
+    if (!timer_set)
+        inode_unref(inode);
+    if (timer_cancelled)
+        inode_unref(inode);
+    /*
+     * We need to invalidate the inode to prevent the kernel from serving
+     * reads from an older cached value despite a change in spb_choice to
+     * a new value.
+     */
+    inode_invalidate(inode);
+out:
+    GF_FREE(data);
+    AFR_STACK_UNWIND(setxattr, frame, ret, op_errno, NULL);
+    return 0;
 }
 
-
-static void
-afr_lookup_self_heal_check (xlator_t *this, afr_local_t *local,
-                            struct iatt *buf, struct iatt *lookup_buf)
+int
+afr_accused_fill(xlator_t *this, dict_t *xdata, unsigned char *accused,
+                 afr_transaction_type type)
 {
-        if (FILETYPE_DIFFERS (buf, lookup_buf)) {
-                /* mismatching filetypes with same name
-                */
+    afr_private_t *priv = NULL;
+    int i = 0;
+    int idx = afr_index_for_transaction_type(type);
+    void *pending_raw = NULL;
+    int pending[3];
+    int ret = 0;
+
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        ret = dict_get_ptr(xdata, priv->pending_key[i], &pending_raw);
+        if (ret) /* no pending flags */
+            continue;
+        memcpy(pending, pending_raw, sizeof(pending));
+
+        if (ntoh32(pending[idx]))
+            accused[i] = 1;
+    }
+
+    return 0;
+}
 
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "filetype differs for %s ", local->loc.path);
+int
+afr_accuse_smallfiles(xlator_t *this, struct afr_reply *replies,
+                      unsigned char *data_accused)
+{
+    int i = 0;
+    afr_private_t *priv = NULL;
+    uint64_t maxsize = 0;
+
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (replies[i].valid && replies[i].xdata &&
+            dict_get_sizen(replies[i].xdata, GLUSTERFS_BAD_INODE))
+            continue;
+        if (data_accused[i])
+            continue;
+        if (replies[i].poststat.ia_size > maxsize)
+            maxsize = replies[i].poststat.ia_size;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (data_accused[i])
+            continue;
+        if (AFR_IS_ARBITER_BRICK(priv, i))
+            continue;
+        if (replies[i].poststat.ia_size < maxsize)
+            data_accused[i] = 1;
+    }
+
+    return 0;
+}
 
-                local->govinda_gOvinda = 1;
+int
+afr_readables_fill(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                   unsigned char *data_accused, unsigned char *metadata_accused,
+                   unsigned char *data_readable,
+                   unsigned char *metadata_readable, struct afr_reply *replies)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    dict_t *xdata = NULL;
+    int i = 0;
+    int ret = 0;
+    ia_type_t ia_type = IA_INVAL;
+
+    local = frame->local;
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        data_readable[i] = 1;
+        metadata_readable[i] = 1;
+    }
+    if (AFR_IS_ARBITER_BRICK(priv, ARBITER_BRICK_INDEX)) {
+        data_readable[ARBITER_BRICK_INDEX] = 0;
+        metadata_readable[ARBITER_BRICK_INDEX] = 0;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (replies) { /* Lookup */
+            if (!replies[i].valid || replies[i].op_ret == -1 ||
+                (replies[i].xdata &&
+                 dict_get_sizen(replies[i].xdata, GLUSTERFS_BAD_INODE))) {
+                data_readable[i] = 0;
+                metadata_readable[i] = 0;
+                continue;
+            }
+
+            xdata = replies[i].xdata;
+            ia_type = replies[i].poststat.ia_type;
+        } else { /* pre-op xattrop */
+            xdata = local->transaction.changelog_xdata[i];
+            ia_type = inode->ia_type;
         }
 
-        if (PERMISSION_DIFFERS (buf, lookup_buf)) {
-                /* mismatching permissions */
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "permissions differ for %s ", local->loc.path);
-                local->self_heal.need_metadata_self_heal = _gf_true;
+        if (!xdata)
+            continue; /* mkdir_cbk sends NULL xdata_rsp. */
+        afr_accused_fill(this, xdata, data_accused,
+                         (ia_type == IA_IFDIR) ? AFR_ENTRY_TRANSACTION
+                                               : AFR_DATA_TRANSACTION);
+
+        afr_accused_fill(this, xdata, metadata_accused,
+                         AFR_METADATA_TRANSACTION);
+    }
+
+    if (replies && ia_type != IA_INVAL && ia_type != IA_IFDIR &&
+        /* We want to accuse small files only when we know for
+         * sure that there is no IO happening. Otherwise, the
+         * ia_sizes obtained in post-refresh replies may
+         * mismatch due to a race between inode-refresh and
+         * ongoing writes, causing spurious heal launches*/
+        !afr_is_possibly_under_txn(AFR_DATA_TRANSACTION, local, this)) {
+        afr_accuse_smallfiles(this, replies, data_accused);
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (data_accused[i]) {
+            data_readable[i] = 0;
+            ret = 1;
         }
-
-        if (OWNERSHIP_DIFFERS (buf, lookup_buf)) {
-                /* mismatching permissions */
-                local->self_heal.need_metadata_self_heal = _gf_true;
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "ownership differs for %s ", local->loc.path);
+        if (metadata_accused[i]) {
+            metadata_readable[i] = 0;
+            ret = 1;
         }
+    }
+    return ret;
+}
 
-        if (SIZE_DIFFERS (buf, lookup_buf)
-            && IA_ISREG (buf->ia_type)) {
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "size differs for %s ", local->loc.path);
-                local->self_heal.need_data_self_heal = _gf_true;
+int
+afr_replies_interpret(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                      gf_boolean_t *start_heal)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    struct afr_reply *replies = NULL;
+    int event_generation = 0;
+    int i = 0;
+    unsigned char *data_accused = NULL;
+    unsigned char *metadata_accused = NULL;
+    unsigned char *data_readable = NULL;
+    unsigned char *metadata_readable = NULL;
+    int ret = 0;
+
+    local = frame->local;
+    priv = this->private;
+    replies = local->replies;
+    event_generation = local->event_generation;
+
+    data_accused = alloca0(priv->child_count);
+    data_readable = alloca0(priv->child_count);
+    metadata_accused = alloca0(priv->child_count);
+    metadata_readable = alloca0(priv->child_count);
+
+    ret = afr_readables_fill(frame, this, inode, data_accused, metadata_accused,
+                             data_readable, metadata_readable, replies);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (start_heal && priv->child_up[i] &&
+            (data_accused[i] || metadata_accused[i])) {
+            *start_heal = _gf_true;
+            break;
         }
-
+    }
+    afr_inode_read_subvol_set(inode, this, data_readable, metadata_readable,
+                              event_generation);
+    return ret;
 }
 
+int
+afr_refresh_selfheal_done(int ret, call_frame_t *heal, void *opaque)
+{
+    if (heal)
+        AFR_STACK_DESTROY(heal);
+    return 0;
+}
 
-static void
-afr_lookup_done (call_frame_t *frame, xlator_t *this, struct iatt *lookup_buf)
+int
+afr_inode_refresh_err(call_frame_t *frame, xlator_t *this)
 {
-        int  unwind = 1;
-        int  source = -1;
-        int  up_count = 0;
-        char sh_type_str[256] = {0,};
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    int err = 0;
+
+    local = frame->local;
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->replies[i].valid && !local->replies[i].op_ret) {
+            err = 0;
+            goto ret;
+        }
+    }
 
-        afr_private_t *priv  = NULL;
-        afr_local_t   *local = NULL;
+    err = afr_final_errno(local, priv);
+ret:
+    return err;
+}
 
-        priv  = this->private;
-        local = frame->local;
+gf_boolean_t
+afr_selfheal_enabled(const xlator_t *this)
+{
+    const afr_private_t *priv = this->private;
 
-        local->cont.lookup.postparent.ia_ino  = local->cont.lookup.parent_ino;
+    return priv->data_self_heal || priv->metadata_self_heal ||
+           priv->entry_self_heal;
+}
 
-        if (local->cont.lookup.ino) {
-                local->cont.lookup.buf.ia_ino = local->cont.lookup.ino;
+int
+afr_txn_refresh_done(call_frame_t *frame, xlator_t *this, int err)
+{
+    call_frame_t *heal_frame = NULL;
+    afr_local_t *heal_local = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    inode_t *inode = NULL;
+    int event_generation = 0;
+    int read_subvol = -1;
+    int ret = 0;
+
+    local = frame->local;
+    inode = local->inode;
+    priv = this->private;
+
+    if (err)
+        goto refresh_done;
+
+    if (local->op == GF_FOP_LOOKUP)
+        goto refresh_done;
+
+    ret = afr_inode_get_readable(frame, inode, this, local->readable,
+                                 &event_generation, local->transaction.type);
+
+    if (ret == -EIO) {
+        /* No readable subvolume even after refresh ==> splitbrain.*/
+        if (!priv->fav_child_policy) {
+            err = EIO;
+            goto refresh_done;
         }
-
-        if (local->op_ret == 0) {
-                /* KLUDGE: assuming DHT will not itransform in
-                   revalidate */
-                if (local->cont.lookup.inode->ino) {
-                        local->cont.lookup.buf.ia_ino =
-                                local->cont.lookup.inode->ino;
-                }
+        read_subvol = afr_sh_get_fav_by_policy(this, local->replies, inode,
+                                               NULL);
+        if (read_subvol == -1) {
+            err = EIO;
+            goto refresh_done;
         }
-        up_count = afr_up_children_count (priv->child_count, priv->child_up);
-        if (up_count == 1) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Only 1 child up - do not attempt to detect self heal");
 
-                goto unwind;
+        heal_frame = afr_frame_create(this, NULL);
+        if (!heal_frame) {
+            err = EIO;
+            goto refresh_done;
         }
-
-        if (local->success_count && local->enoent_count) {
-                local->self_heal.need_metadata_self_heal = _gf_true;
-                local->self_heal.need_data_self_heal     = _gf_true;
-                local->self_heal.need_entry_self_heal    = _gf_true;
-                gf_log(this->name, GF_LOG_NORMAL,
-                       "entries are missing in lookup of %s.",
-                       local->loc.path);
+        heal_local = heal_frame->local;
+        heal_local->xdata_req = dict_new();
+        if (!heal_local->xdata_req) {
+            err = EIO;
+            AFR_STACK_DESTROY(heal_frame);
+            goto refresh_done;
         }
+        heal_local->heal_frame = frame;
+        ret = synctask_new(this->ctx->env, afr_fav_child_reset_sink_xattrs,
+                           afr_fav_child_reset_sink_xattrs_cbk, heal_frame,
+                           heal_frame);
+        return 0;
+    }
 
-        if (local->success_count) {
-                /* check for split-brain case in previous lookup */
-                if (afr_is_split_brain (this,
-                                        local->cont.lookup.inode)) {
-                        local->self_heal.need_data_self_heal = _gf_true;
-                        gf_log(this->name, GF_LOG_NORMAL,
-                               "split brain detected during lookup of "
-                               "%s.", local->loc.path);
-                }
+refresh_done:
+    afr_local_replies_wipe(local, this->private);
+    local->refreshfn(frame, this, err);
+
+    return 0;
+}
+
+int
+afr_inode_refresh_done(call_frame_t *frame, xlator_t *this, int error)
+{
+    call_frame_t *heal_frame = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    gf_boolean_t start_heal = _gf_false;
+    afr_local_t *heal_local = NULL;
+    unsigned char *success_replies = NULL;
+    int ret = 0;
+
+    if (error != 0) {
+        goto refresh_done;
+    }
+
+    local = frame->local;
+    priv = this->private;
+    success_replies = alloca0(priv->child_count);
+    afr_fill_success_replies(local, priv, success_replies);
+
+    if (priv->thin_arbiter_count && local->is_read_txn &&
+        AFR_COUNT(success_replies, priv->child_count) != priv->child_count) {
+        /* We need to query the good bricks and/or thin-arbiter.*/
+        if (success_replies[0]) {
+            local->read_txn_query_child = AFR_CHILD_ZERO;
+        } else if (success_replies[1]) {
+            local->read_txn_query_child = AFR_CHILD_ONE;
+        }
+        error = EINVAL;
+        goto refresh_done;
+    }
+
+    if (!afr_has_quorum(success_replies, this, frame)) {
+        error = afr_final_errno(frame->local, this->private);
+        if (!error)
+            error = afr_quorum_errno(priv);
+        goto refresh_done;
+    }
+
+    ret = afr_replies_interpret(frame, this, local->refreshinode, &start_heal);
+
+    if (ret && afr_selfheal_enabled(this) && start_heal) {
+        heal_frame = afr_frame_create(this, NULL);
+        if (!heal_frame)
+            goto refresh_done;
+        heal_local = heal_frame->local;
+        heal_local->refreshinode = inode_ref(local->refreshinode);
+        heal_local->heal_frame = heal_frame;
+        if (!afr_throttled_selfheal(heal_frame, this)) {
+            AFR_STACK_DESTROY(heal_frame);
+            goto refresh_done;
         }
+    }
 
-        if ((local->self_heal.need_metadata_self_heal
-             || local->self_heal.need_data_self_heal
-             || local->self_heal.need_entry_self_heal)
-            && ((!local->cont.lookup.is_revalidate)
-                || (local->op_ret != -1))) {
+refresh_done:
+    afr_txn_refresh_done(frame, this, error);
 
-                if (local->inodelk_count || local->entrylk_count) {
+    return 0;
+}
 
-                        /* Someone else is doing self-heal on this file.
-                           So just make a best effort to set the read-subvolume
-                           and return */
+void
+afr_inode_refresh_subvol_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                             int op_ret, int op_errno, struct iatt *buf,
+                             dict_t *xdata, struct iatt *par)
+{
+    afr_local_t *local = NULL;
+    int call_child = (long)cookie;
+    int8_t need_heal = 1;
+    int call_count = 0;
+    int ret = 0;
+
+    local = frame->local;
+    local->replies[call_child].valid = 1;
+    local->replies[call_child].op_ret = op_ret;
+    local->replies[call_child].op_errno = op_errno;
+    if (op_ret != -1) {
+        local->replies[call_child].poststat = *buf;
+        if (par)
+            local->replies[call_child].postparent = *par;
+        if (xdata)
+            local->replies[call_child].xdata = dict_ref(xdata);
+    }
+
+    if (xdata) {
+        ret = dict_get_int8(xdata, "link-count", &need_heal);
+        if (ret) {
+            gf_msg_debug(this->name, -ret, "Unable to get link count");
+        }
+    }
+
+    local->replies[call_child].need_heal = need_heal;
+    call_count = afr_frame_return(frame);
+    if (call_count == 0) {
+        afr_set_need_heal(this, local);
+        ret = afr_inode_refresh_err(frame, this);
+        if (ret) {
+            gf_msg_debug(this->name, ret, "afr_inode_refresh_err failed");
+        }
+        afr_inode_refresh_done(frame, this, ret);
+    }
+}
 
-                        if (IA_ISREG (local->cont.lookup.inode->ia_type)) {
-                                source = afr_self_heal_get_source (this, local, local->cont.lookup.xattrs);
+int
+afr_inode_refresh_subvol_with_lookup_cbk(call_frame_t *frame, void *cookie,
+                                         xlator_t *this, int op_ret,
+                                         int op_errno, inode_t *inode,
+                                         struct iatt *buf, dict_t *xdata,
+                                         struct iatt *par)
+{
+    afr_inode_refresh_subvol_cbk(frame, cookie, this, op_ret, op_errno, buf,
+                                 xdata, par);
+    return 0;
+}
 
-                                if (source >= 0) {
-                                        afr_set_read_child (this,
-                                                            local->cont.lookup.inode,
-                                                            source);
-                                }
-                        }
-                } else {
-                        if (!local->cont.lookup.inode->ia_type) {
-                                /* fix for RT #602 */
-                                local->cont.lookup.inode->ia_type =
-                                        lookup_buf->ia_type;
-                        }
+int
+afr_inode_refresh_subvol_with_lookup(call_frame_t *frame, xlator_t *this, int i,
+                                     inode_t *inode, uuid_t gfid, dict_t *xdata)
+{
+    loc_t loc = {
+        0,
+    };
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+
+    loc.inode = inode;
+    if (gf_uuid_is_null(inode->gfid) && gfid) {
+        /* To handle setattr/setxattr on yet to be linked inode from
+         * dht */
+        gf_uuid_copy(loc.gfid, gfid);
+    } else {
+        gf_uuid_copy(loc.gfid, inode->gfid);
+    }
+
+    STACK_WIND_COOKIE(frame, afr_inode_refresh_subvol_with_lookup_cbk,
+                      (void *)(long)i, priv->children[i],
+                      priv->children[i]->fops->lookup, &loc, xdata);
+    return 0;
+}
 
-                        local->self_heal.background = _gf_true;
-                        local->self_heal.type       = local->cont.lookup.buf.ia_type;
-                        local->self_heal.unwind     = afr_self_heal_lookup_unwind;
+int
+afr_inode_refresh_subvol_with_fstat_cbk(call_frame_t *frame, void *cookie,
+                                        xlator_t *this, int32_t op_ret,
+                                        int32_t op_errno, struct iatt *buf,
+                                        dict_t *xdata)
+{
+    afr_inode_refresh_subvol_cbk(frame, cookie, this, op_ret, op_errno, buf,
+                                 xdata, NULL);
+    return 0;
+}
 
-                        unwind = 0;
+int
+afr_inode_refresh_subvol_with_fstat(call_frame_t *frame, xlator_t *this, int i,
+                                    dict_t *xdata)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
 
-                        afr_self_heal_type_str_get(&local->self_heal,
-                                                   sh_type_str,
-                                                   sizeof(sh_type_str));
+    priv = this->private;
+    local = frame->local;
 
-                        gf_log (this->name, GF_LOG_NORMAL, "background %s "
-                                "self-heal triggered. path: %s",
-                                sh_type_str, local->loc.path);
+    STACK_WIND_COOKIE(frame, afr_inode_refresh_subvol_with_fstat_cbk,
+                      (void *)(long)i, priv->children[i],
+                      priv->children[i]->fops->fstat, local->fd, xdata);
+    return 0;
+}
 
-                        afr_self_heal (frame, this);
-                }
+int
+afr_inode_refresh_do(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int call_count = 0;
+    int i = 0;
+    int ret = 0;
+    dict_t *xdata = NULL;
+    afr_fd_ctx_t *fd_ctx = NULL;
+    unsigned char *wind_subvols = NULL;
+
+    priv = this->private;
+    local = frame->local;
+    wind_subvols = alloca0(priv->child_count);
+
+    afr_local_replies_wipe(local, priv);
+
+    if (local->fd) {
+        fd_ctx = afr_fd_ctx_get(local->fd, this);
+        if (!fd_ctx) {
+            afr_inode_refresh_done(frame, this, EINVAL);
+            return 0;
         }
+    }
 
-unwind:
-        if (unwind) {
-                AFR_STACK_UNWIND (lookup, frame, local->op_ret,
-                                  local->op_errno,
-                                  local->cont.lookup.inode,
-                                  &local->cont.lookup.buf,
-                                  local->cont.lookup.xattr,
-                                  &local->cont.lookup.postparent);
-        }
-}
+    xdata = dict_new();
+    if (!xdata) {
+        afr_inode_refresh_done(frame, this, ENOMEM);
+        return 0;
+    }
 
+    ret = afr_xattr_req_prepare(this, xdata);
+    if (ret != 0) {
+        dict_unref(xdata);
+        afr_inode_refresh_done(frame, this, -ret);
+        return 0;
+    }
 
-/*
- * During a lookup, some errors are more "important" than
- * others in that they must be given higher priority while
- * returning to the user.
- *
- * The hierarchy is ESTALE > ENOENT > others
- *
- */
+    ret = dict_set_sizen_str_sizen(xdata, "link-count", GF_XATTROP_INDEX_COUNT);
+    if (ret) {
+        gf_msg_debug(this->name, -ret, "Unable to set link-count in dict ");
+    }
 
-static gf_boolean_t
-__error_more_important (int32_t old_errno, int32_t new_errno)
-{
-        gf_boolean_t ret = _gf_true;
+    ret = dict_set_str_sizen(xdata, GLUSTERFS_INODELK_DOM_COUNT, this->name);
+    if (ret) {
+        gf_msg_debug(this->name, -ret,
+                     "Unable to set inodelk-dom-count in dict ");
+    }
+
+    if (local->fd) {
+        for (i = 0; i < priv->child_count; i++) {
+            if (local->child_up[i] && fd_ctx->opened_on[i] == AFR_FD_OPENED)
+                wind_subvols[i] = 1;
+        }
+    } else {
+        memcpy(wind_subvols, local->child_up,
+               sizeof(*local->child_up) * priv->child_count);
+    }
+
+    local->call_count = AFR_COUNT(wind_subvols, priv->child_count);
+
+    call_count = local->call_count;
+    if (!call_count) {
+        dict_unref(xdata);
+        if (local->fd && AFR_COUNT(local->child_up, priv->child_count))
+            afr_inode_refresh_done(frame, this, EBADFD);
+        else
+            afr_inode_refresh_done(frame, this, ENOTCONN);
+        return 0;
+    }
+    for (i = 0; i < priv->child_count; i++) {
+        if (!wind_subvols[i])
+            continue;
 
-        /* Nothing should ever overwrite ESTALE */
-        if (old_errno == ESTALE)
-                ret = _gf_false;
+        if (local->fd)
+            afr_inode_refresh_subvol_with_fstat(frame, this, i, xdata);
+        else
+            afr_inode_refresh_subvol_with_lookup(
+                frame, this, i, local->refreshinode, local->refreshgfid, xdata);
 
-        /* Nothing should overwrite ENOENT, except ESTALE */
-        else if ((old_errno == ENOENT) && (new_errno != ESTALE))
-                ret = _gf_false;
+        if (!--call_count)
+            break;
+    }
 
-        return ret;
-}
+    dict_unref(xdata);
 
+    return 0;
+}
 
 int
-afr_fresh_lookup_cbk (call_frame_t *frame, void *cookie,
-                      xlator_t *this,  int32_t op_ret,  int32_t op_errno,
-                      inode_t *inode,   struct iatt *buf, dict_t *xattr,
-                      struct iatt *postparent)
+afr_inode_refresh(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                  uuid_t gfid, afr_inode_refresh_cbk_t refreshfn)
 {
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
-        struct iatt *   lookup_buf = NULL;
+    afr_local_t *local = NULL;
 
-        int             call_count      = -1;
-        int             child_index     = -1;
-        int             first_up_child  = -1;
+    local = frame->local;
 
-        child_index = (long) cookie;
-        priv = this->private;
+    local->refreshfn = refreshfn;
 
-        LOCK (&frame->lock);
-        {
-                local = frame->local;
+    if (local->refreshinode) {
+        inode_unref(local->refreshinode);
+        local->refreshinode = NULL;
+    }
 
-                lookup_buf = &local->cont.lookup.buf;
+    local->refreshinode = inode_ref(inode);
 
-                if (op_ret == -1) {
-                        if (op_errno == ENOENT)
-                                local->enoent_count++;
+    if (gfid)
+        gf_uuid_copy(local->refreshgfid, gfid);
+    else
+        gf_uuid_clear(local->refreshgfid);
 
-                        if (__error_more_important (local->op_errno, op_errno))
-                                local->op_errno = op_errno;
+    afr_inode_refresh_do(frame, this);
 
-                        if (local->op_errno == ESTALE) {
-                                local->op_ret = -1;
-                        }
+    return 0;
+}
 
-                        goto unlock;
-                }
+int
+afr_xattr_req_prepare(xlator_t *this, dict_t *xattr_req)
+{
+    int i = 0;
+    afr_private_t *priv = NULL;
+    int ret = 0;
 
-                afr_lookup_collect_xattr (local, this, child_index, xattr);
+    priv = this->private;
 
-                first_up_child = afr_first_up_child (priv);
+    for (i = 0; i < priv->child_count; i++) {
+        ret = dict_set_uint64(xattr_req, priv->pending_key[i],
+                              AFR_NUM_CHANGE_LOGS * sizeof(int));
+        if (ret < 0)
+            gf_msg(this->name, GF_LOG_WARNING, -ret, AFR_MSG_DICT_SET_FAILED,
+                   "Unable to set dict value for %s", priv->pending_key[i]);
+        /* 3 = data+metadata+entry */
+    }
+    ret = dict_set_uint64(xattr_req, AFR_DIRTY,
+                          AFR_NUM_CHANGE_LOGS * sizeof(int));
+    if (ret) {
+        gf_msg_debug(this->name, -ret,
+                     "failed to set dirty "
+                     "query flag");
+    }
+
+    ret = dict_set_int32_sizen(xattr_req, "list-xattr", 1);
+    if (ret) {
+        gf_msg_debug(this->name, -ret, "Unable to set list-xattr in dict ");
+    }
+
+    return ret;
+}
 
-                if (child_index == first_up_child) {
-                        local->cont.lookup.ino =
-                                afr_itransform (buf->ia_ino,
-                                                priv->child_count,
-                                                first_up_child);
-                }
+int
+afr_lookup_xattr_req_prepare(afr_local_t *local, xlator_t *this,
+                             dict_t *xattr_req, loc_t *loc)
+{
+    int ret = -ENOMEM;
+
+    if (!local->xattr_req)
+        local->xattr_req = dict_new();
+
+    if (!local->xattr_req)
+        goto out;
+
+    if (xattr_req && (xattr_req != local->xattr_req))
+        dict_copy(xattr_req, local->xattr_req);
+
+    ret = afr_xattr_req_prepare(this, local->xattr_req);
+
+    ret = dict_set_uint64(local->xattr_req, GLUSTERFS_INODELK_COUNT, 0);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, AFR_MSG_DICT_SET_FAILED,
+               "%s: Unable to set dict value for %s", loc->path,
+               GLUSTERFS_INODELK_COUNT);
+    }
+    ret = dict_set_uint64(local->xattr_req, GLUSTERFS_ENTRYLK_COUNT, 0);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, AFR_MSG_DICT_SET_FAILED,
+               "%s: Unable to set dict value for %s", loc->path,
+               GLUSTERFS_ENTRYLK_COUNT);
+    }
+
+    ret = dict_set_uint32(local->xattr_req, GLUSTERFS_PARENT_ENTRYLK, 0);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, AFR_MSG_DICT_SET_FAILED,
+               "%s: Unable to set dict value for %s", loc->path,
+               GLUSTERFS_PARENT_ENTRYLK);
+    }
+
+    ret = dict_set_sizen_str_sizen(local->xattr_req, "link-count",
+                                   GF_XATTROP_INDEX_COUNT);
+    if (ret) {
+        gf_msg_debug(this->name, -ret, "Unable to set link-count in dict ");
+    }
+
+    ret = 0;
+out:
+    return ret;
+}
 
-                if (local->success_count == 0) {
-                        if (local->op_errno != ESTALE)
-                                local->op_ret = op_ret;
-
-                        local->cont.lookup.inode               = inode_ref (inode);
-                        local->cont.lookup.xattr               = dict_ref (xattr);
-                        local->cont.lookup.xattrs[child_index] = dict_ref (xattr);
-                        local->cont.lookup.postparent          = *postparent;
-
-                        if (priv->first_lookup && inode->ino == 1) {
-                                gf_log (this->name, GF_LOG_NORMAL,
-                                        "added root inode");
-                                priv->root_inode = inode_ref (inode);
-                                priv->first_lookup = 0;
-                        }
-
-                        *lookup_buf = *buf;
-
-                        lookup_buf->ia_ino = afr_itransform (buf->ia_ino,
-                                                             priv->child_count,
-                                                             child_index);
-                        if (priv->read_child >= 0) {
-                                afr_set_read_child (this,
-                                                    local->cont.lookup.inode,
-                                                    priv->read_child);
-                        } else {
-                                afr_set_read_child (this,
-                                                    local->cont.lookup.inode,
-                                                    child_index);
-                        }
-
-                } else {
-                        afr_lookup_self_heal_check (this, local, buf, lookup_buf);
-
-                        if (child_index == local->read_child_index) {
-                                /*
-                                   lookup has succeeded on the read child.
-                                   So use its inode number
-                                */
-                                if (local->cont.lookup.xattr)
-                                        dict_unref (local->cont.lookup.xattr);
-
-                                local->cont.lookup.xattr = dict_ref (xattr);
-                                local->cont.lookup.xattrs[child_index] = dict_ref (xattr);
-                                local->cont.lookup.postparent          = *postparent;
-
-                                *lookup_buf = *buf;
-
-                                if (priv->read_child >= 0) {
-                                        afr_set_read_child (this,
-                                                            local->cont.lookup.inode,
-                                                            priv->read_child);
-                                } else {
-                                        afr_set_read_child (this,
-                                                            local->cont.lookup.inode,
-                                                            local->read_child_index);
-                                }
-                        }
+int
+afr_least_pending_reads_child(afr_private_t *priv, unsigned char *readable)
+{
+    int i = 0;
+    int child = -1;
+    int64_t read_iter = -1;
+    int64_t pending_read = -1;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (AFR_IS_ARBITER_BRICK(priv, i) || !readable[i])
+            continue;
+        read_iter = GF_ATOMIC_GET(priv->pending_reads[i]);
+        if (child == -1 || read_iter < pending_read) {
+            pending_read = read_iter;
+            child = i;
+        }
+    }
 
-                }
+    return child;
+}
 
-                local->success_count++;
-        }
-unlock:
-        UNLOCK (&frame->lock);
+static int32_t
+afr_least_latency_child(afr_private_t *priv, unsigned char *readable)
+{
+    int32_t i = 0;
+    int child = -1;
 
-        call_count = afr_frame_return (frame);
+    for (i = 0; i < priv->child_count; i++) {
+        if (AFR_IS_ARBITER_BRICK(priv, i) || !readable[i] ||
+            priv->child_latency[i] < 0)
+            continue;
 
-        if (call_count == 0) {
-                afr_lookup_done (frame, this, lookup_buf);
+        if (child == -1 ||
+            priv->child_latency[i] < priv->child_latency[child]) {
+            child = i;
         }
-
-        return 0;
+    }
+    return child;
 }
 
+static int32_t
+afr_least_latency_times_pending_reads_child(afr_private_t *priv,
+                                            unsigned char *readable)
+{
+    int32_t i = 0;
+    int child = -1;
+    int64_t pending_read = 0;
+    int64_t latency = -1;
+    int64_t least_latency = -1;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (AFR_IS_ARBITER_BRICK(priv, i) || !readable[i] ||
+            priv->child_latency[i] < 0)
+            continue;
+
+        pending_read = GF_ATOMIC_GET(priv->pending_reads[i]);
+        latency = (pending_read + 1) * priv->child_latency[i];
+
+        if (child == -1 || latency < least_latency) {
+            least_latency = latency;
+            child = i;
+        }
+    }
+    return child;
+}
 
 int
-afr_revalidate_lookup_cbk (call_frame_t *frame, void *cookie,
-                           xlator_t *this, int32_t op_ret, int32_t op_errno,
-                           inode_t *inode, struct iatt *buf, dict_t *xattr,
-                           struct iatt *postparent)
+afr_hash_child(afr_read_subvol_args_t *args, afr_private_t *priv,
+               unsigned char *readable)
 {
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
-        struct iatt *   lookup_buf = NULL;
-
-        int             call_count      = -1;
-        int             child_index     = -1;
-        int             first_up_child  = -1;
-
-        child_index = (long) cookie;
-        priv = this->private;
+    uuid_t gfid_copy = {
+        0,
+    };
+    pid_t pid;
+    int child = -1;
+
+    switch (priv->hash_mode) {
+        case AFR_READ_POLICY_FIRST_UP:
+            break;
+        case AFR_READ_POLICY_GFID_HASH:
+            gf_uuid_copy(gfid_copy, args->gfid);
+            child = SuperFastHash((char *)gfid_copy, sizeof(gfid_copy)) %
+                    priv->child_count;
+            break;
+        case AFR_READ_POLICY_GFID_PID_HASH:
+            if (args->ia_type != IA_IFDIR) {
+                /*
+                 * Why getpid?  Because it's one of the cheapest calls
+                 * available - faster than gethostname etc. - and
+                 * returns a constant-length value that's sure to be
+                 * shorter than a UUID. It's still very unlikely to be
+                 * the same across clients, so it still provides good
+                 * mixing.  We're not trying for perfection here. All we
+                 * need is a low probability that multiple clients
+                 * won't converge on the same subvolume.
+                 */
+                gf_uuid_copy(gfid_copy, args->gfid);
+                pid = getpid();
+                *(pid_t *)gfid_copy ^= pid;
+            }
+            child = SuperFastHash((char *)gfid_copy, sizeof(gfid_copy)) %
+                    priv->child_count;
+            break;
+        case AFR_READ_POLICY_LESS_LOAD:
+            child = afr_least_pending_reads_child(priv, readable);
+            break;
+        case AFR_READ_POLICY_LEAST_LATENCY:
+            child = afr_least_latency_child(priv, readable);
+            break;
+        case AFR_READ_POLICY_LOAD_LATENCY_HYBRID:
+            child = afr_least_latency_times_pending_reads_child(priv, readable);
+            break;
+    }
+
+    return child;
+}
 
-        LOCK (&frame->lock);
-        {
-                local = frame->local;
+int
+afr_read_subvol_select_by_policy(inode_t *inode, xlator_t *this,
+                                 unsigned char *readable,
+                                 afr_read_subvol_args_t *args)
+{
+    int i = 0;
+    int read_subvol = -1;
+    afr_private_t *priv = NULL;
+    afr_read_subvol_args_t local_args = {
+        0,
+    };
+
+    priv = this->private;
+
+    /* first preference - explicitly specified or local subvolume */
+    if (priv->read_child >= 0 && readable[priv->read_child])
+        return priv->read_child;
+
+    if (inode_is_linked(inode)) {
+        gf_uuid_copy(local_args.gfid, inode->gfid);
+        local_args.ia_type = inode->ia_type;
+    } else if (args) {
+        local_args = *args;
+    }
+
+    /* second preference - use hashed mode */
+    read_subvol = afr_hash_child(&local_args, priv, readable);
+    if (read_subvol >= 0 && readable[read_subvol])
+        return read_subvol;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (readable[i])
+            return i;
+    }
+
+    /* no readable subvolumes, either split brain or all subvols down */
+
+    return -1;
+}
 
-                lookup_buf = &local->cont.lookup.buf;
+int
+afr_inode_read_subvol_type_get(inode_t *inode, xlator_t *this,
+                               unsigned char *readable, int *event_p, int type)
+{
+    int ret = -1;
 
-                if (op_ret == -1) {
-                        if (op_errno == ENOENT)
-                                local->enoent_count++;
+    if (type == AFR_METADATA_TRANSACTION)
+        ret = afr_inode_read_subvol_get(inode, this, 0, readable, event_p);
+    else
+        ret = afr_inode_read_subvol_get(inode, this, readable, 0, event_p);
+    return ret;
+}
 
-                        if (__error_more_important (local->op_errno, op_errno))
-                            local->op_errno = op_errno;
+void
+afr_readables_intersect_get(inode_t *inode, xlator_t *this, int *event,
+                            unsigned char *intersection)
+{
+    afr_private_t *priv = NULL;
+    unsigned char *data_readable = NULL;
+    unsigned char *metadata_readable = NULL;
+    unsigned char *intersect = NULL;
+
+    priv = this->private;
+    data_readable = alloca0(priv->child_count);
+    metadata_readable = alloca0(priv->child_count);
+    intersect = alloca0(priv->child_count);
+
+    afr_inode_read_subvol_get(inode, this, data_readable, metadata_readable,
+                              event);
+
+    AFR_INTERSECT(intersect, data_readable, metadata_readable,
+                  priv->child_count);
+    if (intersection)
+        memcpy(intersection, intersect,
+               sizeof(*intersection) * priv->child_count);
+}
 
-                            if (local->op_errno == ESTALE) {
-                                    local->op_ret = -1;
-                            }
+int
+afr_read_subvol_get(inode_t *inode, xlator_t *this, int *subvol_p,
+                    unsigned char *readables, int *event_p,
+                    afr_transaction_type type, afr_read_subvol_args_t *args)
+{
+    afr_private_t *priv = NULL;
+    unsigned char *readable = NULL;
+    unsigned char *intersection = NULL;
+    int subvol = -1;
+    int event = 0;
+
+    priv = this->private;
+
+    readable = alloca0(priv->child_count);
+    intersection = alloca0(priv->child_count);
+
+    afr_inode_read_subvol_type_get(inode, this, readable, &event, type);
+
+    afr_readables_intersect_get(inode, this, &event, intersection);
+
+    if (AFR_COUNT(intersection, priv->child_count) > 0)
+        subvol = afr_read_subvol_select_by_policy(inode, this, intersection,
+                                                  args);
+    else
+        subvol = afr_read_subvol_select_by_policy(inode, this, readable, args);
+    if (subvol_p)
+        *subvol_p = subvol;
+    if (event_p)
+        *event_p = event;
+    if (readables)
+        memcpy(readables, readable, sizeof(*readables) * priv->child_count);
+    return subvol;
+}
 
-                            goto unlock;
-                }
+void
+afr_local_transaction_cleanup(afr_local_t *local, xlator_t *this)
+{
+    afr_private_t *priv = NULL;
+    int i = 0;
 
-                afr_lookup_collect_xattr (local, this, child_index, xattr);
+    priv = this->private;
 
-                first_up_child = afr_first_up_child (priv);
+    afr_matrix_cleanup(local->pending, priv->child_count);
 
-                if (child_index == first_up_child) {
-                        local->cont.lookup.ino =
-                                afr_itransform (buf->ia_ino,
-                                                priv->child_count,
-                                                first_up_child);
-                }
+    GF_FREE(local->internal_lock.lower_locked_nodes);
 
-                /* in case of revalidate, we need to send stat of the
-                 * child whose stat was sent during the first lookup.
-                 * (so that time stamp does not vary with revalidate.
-                 * in case it is down, stat of the fist success will
-                 * be replied */
-
-                /* inode number should be preserved across revalidates */
-
-                if (local->success_count == 0) {
-                        if (local->op_errno != ESTALE)
-                                local->op_ret = op_ret;
-
-                        local->cont.lookup.inode               = inode_ref (inode);
-                        local->cont.lookup.xattr               = dict_ref (xattr);
-                        local->cont.lookup.xattrs[child_index] = dict_ref (xattr);
-                        local->cont.lookup.postparent          = *postparent;
-
-                        *lookup_buf = *buf;
-
-                        lookup_buf->ia_ino = afr_itransform (buf->ia_ino,
-                                                             priv->child_count,
-                                                             child_index);
-
-                        if (priv->read_child >= 0) {
-                                afr_set_read_child (this,
-                                                    local->cont.lookup.inode,
-                                                    priv->read_child);
-                        } else {
-                                afr_set_read_child (this,
-                                                    local->cont.lookup.inode,
-                                                    child_index);
-                        }
-
-                } else {
-                        afr_lookup_self_heal_check (this, local, buf, lookup_buf);
-
-                        if (child_index == local->read_child_index) {
-
-                                /*
-                                   lookup has succeeded on the read child.
-                                   So use its inode number
-                                */
-
-                                if (local->cont.lookup.xattr)
-                                        dict_unref (local->cont.lookup.xattr);
-
-                                local->cont.lookup.xattr               = dict_ref (xattr);
-                                local->cont.lookup.xattrs[child_index] = dict_ref (xattr);
-                                local->cont.lookup.postparent          = *postparent;
-
-                                *lookup_buf = *buf;
-
-                                if (priv->read_child >= 0) {
-                                        afr_set_read_child (this,
-                                                            local->cont.lookup.inode,
-                                                            priv->read_child);
-                                } else {
-                                        afr_set_read_child (this,
-                                                            local->cont.lookup.inode,
-                                                            local->read_child_index);
-                                }
-                        }
+    afr_lockees_cleanup(&local->internal_lock);
 
-                }
+    GF_FREE(local->transaction.pre_op);
 
-                local->success_count++;
+    GF_FREE(local->transaction.pre_op_sources);
+    if (local->transaction.changelog_xdata) {
+        for (i = 0; i < priv->child_count; i++) {
+            if (!local->transaction.changelog_xdata[i])
+                continue;
+            dict_unref(local->transaction.changelog_xdata[i]);
         }
-unlock:
-        UNLOCK (&frame->lock);
+        GF_FREE(local->transaction.changelog_xdata);
+    }
 
-        call_count = afr_frame_return (frame);
+    GF_FREE(local->transaction.failed_subvols);
 
-        if (call_count == 0) {
-                afr_lookup_done (frame, this, lookup_buf);
-        }
+    GF_FREE(local->transaction.basename);
+    GF_FREE(local->transaction.new_basename);
 
-        return 0;
+    loc_wipe(&local->transaction.parent_loc);
+    loc_wipe(&local->transaction.new_parent_loc);
 }
 
+void
+afr_reply_wipe(struct afr_reply *reply)
+{
+    if (reply->xdata) {
+        dict_unref(reply->xdata);
+        reply->xdata = NULL;
+    }
+
+    if (reply->xattr) {
+        dict_unref(reply->xattr);
+        reply->xattr = NULL;
+    }
+}
 
-int
-afr_lookup (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, dict_t *xattr_req)
+void
+afr_replies_wipe(struct afr_reply *replies, int count)
 {
-        afr_private_t *priv = NULL;
-        afr_local_t   *local = NULL;
-        int            ret = -1;
-        int            i = 0;
+    int i = 0;
 
-        fop_lookup_cbk_t callback;
+    for (i = 0; i < count; i++) {
+        afr_reply_wipe(&replies[i]);
+    }
+}
 
-        int call_count = 0;
+void
+afr_local_replies_wipe(afr_local_t *local, afr_private_t *priv)
+{
+    if (!local->replies)
+        return;
 
-        uint64_t       ctx;
+    afr_replies_wipe(local->replies, priv->child_count);
 
-        int32_t        op_errno = 0;
+    memset(local->replies, 0, sizeof(*local->replies) * priv->child_count);
+}
 
-        priv = this->private;
+static gf_boolean_t
+afr_fop_lock_is_unlock(call_frame_t *frame)
+{
+    afr_local_t *local = frame->local;
+    switch (local->op) {
+        case GF_FOP_INODELK:
+        case GF_FOP_FINODELK:
+            if ((F_UNLCK == local->cont.inodelk.in_flock.l_type) &&
+                (local->cont.inodelk.in_cmd == F_SETLKW ||
+                 local->cont.inodelk.in_cmd == F_SETLK))
+                return _gf_true;
+            break;
+        case GF_FOP_ENTRYLK:
+        case GF_FOP_FENTRYLK:
+            if (ENTRYLK_UNLOCK == local->cont.entrylk.in_cmd)
+                return _gf_true;
+            break;
+        default:
+            return _gf_false;
+    }
+    return _gf_false;
+}
 
-        ALLOC_OR_GOTO (local, afr_local_t, out);
+static gf_boolean_t
+afr_lk_is_unlock(int32_t cmd, struct gf_flock *flock)
+{
+    switch (cmd) {
+        case F_RESLK_UNLCK:
+            return _gf_true;
+            break;
 
-        local->op_ret = -1;
+#if F_SETLKW != F_SETLKW64
+        case F_SETLKW64:
+#endif
+        case F_SETLKW:
 
-        frame->local = local;
+#if F_SETLK != F_SETLK64
+        case F_SETLK64:
+#endif
+        case F_SETLK:
+            if (F_UNLCK == flock->l_type)
+                return _gf_true;
+            break;
+        default:
+            return _gf_false;
+    }
+    return _gf_false;
+}
 
-        if (!strcmp (loc->path, "/" GF_REPLICATE_TRASH_DIR)) {
-                op_errno = ENOENT;
-                goto out;
-        }
+void
+afr_handle_inconsistent_fop(call_frame_t *frame, int32_t *op_ret,
+                            int32_t *op_errno)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
 
-        loc_copy (&local->loc, loc);
+    if (!frame || !frame->this || !frame->local || !frame->this->private)
+        return;
 
-        ret = inode_ctx_get (loc->inode, this, &ctx);
-        if (ret == 0) {
-                /* lookup is a revalidate */
+    if (*op_ret < 0)
+        return;
 
-                callback = afr_revalidate_lookup_cbk;
+    /* Failing inodelk/entrylk/lk here is not a good idea because we
+     * need to cleanup the locks on the other bricks if we choose to fail
+     * the fop here. The brick may go down just after unwind happens as well
+     * so anyways the fop will fail when the next fop is sent so leaving
+     * it like this for now.*/
+    local = frame->local;
+    switch (local->op) {
+        case GF_FOP_LOOKUP:
+        case GF_FOP_INODELK:
+        case GF_FOP_FINODELK:
+        case GF_FOP_ENTRYLK:
+        case GF_FOP_FENTRYLK:
+        case GF_FOP_LK:
+            return;
+        default:
+            break;
+    }
 
-                local->cont.lookup.is_revalidate = _gf_true;
-                local->read_child_index          = afr_read_child (this,
-                                                                   loc->inode);
-        } else {
-                callback = afr_fresh_lookup_cbk;
+    priv = frame->this->private;
+    if (!priv->consistent_io)
+        return;
 
-                LOCK (&priv->read_child_lock);
-                {
-                        local->read_child_index = (++priv->read_child_rr)
-                                % (priv->child_count);
-                }
-                UNLOCK (&priv->read_child_lock);
-        }
+    if (local->event_generation &&
+        (local->event_generation != priv->event_generation))
+        goto inconsistent;
 
-        if (loc->parent)
-                local->cont.lookup.parent_ino = loc->parent->ino;
+    return;
+inconsistent:
+    *op_ret = -1;
+    *op_errno = ENOTCONN;
+}
 
-        local->child_up = memdup (priv->child_up, priv->child_count);
+void
+afr_local_cleanup(afr_local_t *local, xlator_t *this)
+{
+    afr_private_t *priv = NULL;
 
-        local->cont.lookup.xattrs = GF_CALLOC (priv->child_count,
-                                    sizeof (*local->cont.lookup.xattr),
-                                    gf_afr_mt_dict_t);
+    if (!local)
+        return;
 
-        local->call_count = afr_up_children_count (priv->child_count,
-                                                   local->child_up);
-        call_count = local->call_count;
+    syncbarrier_destroy(&local->barrier);
 
-        if (local->call_count == 0) {
-                ret      = -1;
-                op_errno = ENOTCONN;
-                goto out;
-        }
+    afr_local_transaction_cleanup(local, this);
 
-        /* By default assume ENOTCONN. On success it will be set to 0. */
-        local->op_errno = ENOTCONN;
+    priv = this->private;
 
-        if (xattr_req == NULL)
-                local->xattr_req = dict_new ();
-        else
-                local->xattr_req = dict_ref (xattr_req);
+    loc_wipe(&local->loc);
+    loc_wipe(&local->newloc);
 
-        for (i = 0; i < priv->child_count; i++) {
-                ret = dict_set_uint64 (local->xattr_req, priv->pending_key[i],
-                                       3 * sizeof(int32_t));
-                if (ret < 0)
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "Unable to set dict value.");
-                /* 3 = data+metadata+entry */
-        }
+    if (local->fd)
+        fd_unref(local->fd);
 
-        ret = dict_set_uint64 (local->xattr_req, GLUSTERFS_INODELK_COUNT, 0);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Unable to set dict value.");
-        }
+    if (local->xattr_req)
+        dict_unref(local->xattr_req);
 
-        ret = dict_set_uint64 (local->xattr_req, GLUSTERFS_ENTRYLK_COUNT, 0);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Unable to set dict value.");
-        }
+    if (local->xattr_rsp)
+        dict_unref(local->xattr_rsp);
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, callback, (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->lookup,
-                                           loc, local->xattr_req);
-                        if (!--call_count)
-                                break;
-                }
-        }
+    if (local->dict)
+        dict_unref(local->dict);
 
-        ret = 0;
-out:
-        if (ret == -1)
-                AFR_STACK_UNWIND (lookup, frame, -1, op_errno,
-                                  NULL, NULL, NULL, NULL);
+    afr_local_replies_wipe(local, priv);
+    GF_FREE(local->replies);
 
-        return 0;
-}
+    GF_FREE(local->child_up);
 
+    GF_FREE(local->read_attempted);
+
+    GF_FREE(local->readable);
+    GF_FREE(local->readable2);
+
+    if (local->inode)
+        inode_unref(local->inode);
 
-/* {{{ open */
+    if (local->parent)
+        inode_unref(local->parent);
+
+    if (local->parent2)
+        inode_unref(local->parent2);
+
+    if (local->refreshinode)
+        inode_unref(local->refreshinode);
+
+    { /* getxattr */
+        GF_FREE(local->cont.getxattr.name);
+    }
+
+    { /* lk */
+        GF_FREE(local->cont.lk.locked_nodes);
+        GF_FREE(local->cont.lk.dom_locked_nodes);
+        GF_FREE(local->cont.lk.dom_lock_op_ret);
+        GF_FREE(local->cont.lk.dom_lock_op_errno);
+    }
+
+    { /* create */
+        if (local->cont.create.fd)
+            fd_unref(local->cont.create.fd);
+        if (local->cont.create.params)
+            dict_unref(local->cont.create.params);
+    }
+
+    { /* mknod */
+        if (local->cont.mknod.params)
+            dict_unref(local->cont.mknod.params);
+    }
+
+    { /* mkdir */
+        if (local->cont.mkdir.params)
+            dict_unref(local->cont.mkdir.params);
+    }
+
+    { /* symlink */
+        if (local->cont.symlink.params)
+            dict_unref(local->cont.symlink.params);
+    }
+
+    { /* writev */
+        GF_FREE(local->cont.writev.vector);
+        if (local->cont.writev.iobref)
+            iobref_unref(local->cont.writev.iobref);
+    }
+
+    { /* setxattr */
+        if (local->cont.setxattr.dict)
+            dict_unref(local->cont.setxattr.dict);
+    }
+
+    { /* fsetxattr */
+        if (local->cont.fsetxattr.dict)
+            dict_unref(local->cont.fsetxattr.dict);
+    }
+
+    { /* removexattr */
+        GF_FREE(local->cont.removexattr.name);
+    }
+    { /* xattrop */
+        if (local->cont.xattrop.xattr)
+            dict_unref(local->cont.xattrop.xattr);
+    }
+    { /* symlink */
+        GF_FREE(local->cont.symlink.linkpath);
+    }
+
+    { /* opendir */
+        GF_FREE(local->cont.opendir.checksum);
+    }
+
+    { /* open */
+        if (local->cont.open.fd)
+            fd_unref(local->cont.open.fd);
+    }
+
+    { /* readdirp */
+        if (local->cont.readdir.dict)
+            dict_unref(local->cont.readdir.dict);
+    }
+
+    { /* inodelk */
+        GF_FREE(local->cont.inodelk.volume);
+        if (local->cont.inodelk.xdata)
+            dict_unref(local->cont.inodelk.xdata);
+    }
+
+    { /* entrylk */
+        GF_FREE(local->cont.entrylk.volume);
+        GF_FREE(local->cont.entrylk.basename);
+        if (local->cont.entrylk.xdata)
+            dict_unref(local->cont.entrylk.xdata);
+    }
+
+    if (local->xdata_req)
+        dict_unref(local->xdata_req);
+
+    if (local->xdata_rsp)
+        dict_unref(local->xdata_rsp);
+}
 
 int
-afr_fd_ctx_set (xlator_t *this, fd_t *fd)
+afr_frame_return(call_frame_t *frame)
 {
-        afr_private_t * priv = NULL;
+    afr_local_t *local = NULL;
+    int call_count = 0;
 
-        int ret    = -1;
+    local = frame->local;
 
-        uint64_t       ctx;
-        afr_fd_ctx_t * fd_ctx = NULL;
+    LOCK(&frame->lock);
+    {
+        call_count = --local->call_count;
+    }
+    UNLOCK(&frame->lock);
 
-        VALIDATE_OR_GOTO (this->private, out);
-        VALIDATE_OR_GOTO (fd, out);
+    return call_count;
+}
 
-        priv = this->private;
+static char *afr_ignore_xattrs[] = {GF_SELINUX_XATTR_KEY, QUOTA_SIZE_KEY, NULL};
 
-        LOCK (&fd->lock);
-        {
-                ret = __fd_ctx_get (fd, this, &ctx);
+gf_boolean_t
+afr_is_xattr_ignorable(char *key)
+{
+    int i = 0;
+
+    if (!strncmp(key, AFR_XATTR_PREFIX, SLEN(AFR_XATTR_PREFIX)))
+        return _gf_true;
+    for (i = 0; afr_ignore_xattrs[i]; i++) {
+        if (!strcmp(key, afr_ignore_xattrs[i]))
+            return _gf_true;
+    }
+    return _gf_false;
+}
 
-                if (ret == 0)
-                        goto unlock;
+static gf_boolean_t
+afr_xattr_match_needed(dict_t *this, char *key1, data_t *value1, void *data)
+{
+    /* Ignore all non-disk (i.e. virtual) xattrs right away. */
+    if (!gf_is_valid_xattr_namespace(key1))
+        return _gf_false;
 
-                fd_ctx = GF_CALLOC (1, sizeof (afr_fd_ctx_t),
-                                    gf_afr_mt_afr_fd_ctx_t);
-                if (!fd_ctx) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Out of memory");
+    /* Ignore on-disk xattrs that AFR doesn't need to heal. */
+    if (!afr_is_xattr_ignorable(key1))
+        return _gf_true;
 
-                        ret = -ENOMEM;
-                        goto unlock;
-                }
+    return _gf_false;
+}
 
-                fd_ctx->pre_op_done = GF_CALLOC (sizeof (*fd_ctx->pre_op_done),
-                                                 priv->child_count,
-                                                 gf_afr_mt_char);
-                if (!fd_ctx->pre_op_done) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Out of memory");
-                        ret = -ENOMEM;
-                        goto unlock;
-                }
+gf_boolean_t
+afr_xattrs_are_equal(dict_t *dict1, dict_t *dict2)
+{
+    return are_dicts_equal(dict1, dict2, afr_xattr_match_needed, NULL);
+}
 
-                fd_ctx->pre_op_piggyback = GF_CALLOC (sizeof (*fd_ctx->pre_op_piggyback),
-                                                      priv->child_count,
-                                                      gf_afr_mt_char);
-                if (!fd_ctx->pre_op_piggyback) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Out of memory");
-                        ret = -ENOMEM;
-                        goto unlock;
-                }
+static int
+afr_get_parent_read_subvol(xlator_t *this, inode_t *parent,
+                           struct afr_reply *replies, unsigned char *readable)
+{
+    int i = 0;
+    int par_read_subvol = -1;
+    int par_read_subvol_iter = -1;
+    afr_private_t *priv = NULL;
 
-                fd_ctx->opened_on = GF_CALLOC (sizeof (*fd_ctx->opened_on),
-                                               priv->child_count,
-                                               gf_afr_mt_char);
-                if (!fd_ctx->opened_on) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Out of memory");
-                        ret = -ENOMEM;
-                        goto unlock;
-                }
+    priv = this->private;
 
-                fd_ctx->up_count   = priv->up_count;
-                fd_ctx->down_count = priv->down_count;
-
-                fd_ctx->locked_on = GF_CALLOC (sizeof (*fd_ctx->locked_on),
-                                               priv->child_count,
-                                               gf_afr_mt_char);
-                if (!fd_ctx->locked_on) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Out of memory");
-                        ret = -ENOMEM;
-                        goto unlock;
-                }
+    if (parent)
+        par_read_subvol = afr_data_subvol_get(parent, this, NULL, NULL, NULL,
+                                              NULL);
 
-                ret = __fd_ctx_set (fd, this, (uint64_t)(long) fd_ctx);
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid)
+            continue;
 
-                INIT_LIST_HEAD (&fd_ctx->entries);
+        if (replies[i].op_ret < 0)
+            continue;
+
+        if (par_read_subvol_iter == -1) {
+            par_read_subvol_iter = i;
+            continue;
         }
-unlock:
-        UNLOCK (&fd->lock);
-out:
-        return ret;
-}
 
-/* {{{ flush */
+        if ((par_read_subvol_iter != par_read_subvol) && readable[i])
+            par_read_subvol_iter = i;
+
+        if (i == par_read_subvol)
+            par_read_subvol_iter = i;
+    }
+    /* At the end of the for-loop, the only reason why @par_read_subvol_iter
+     * could be -1 is when this LOOKUP has failed on all sub-volumes.
+     * So it is okay to send an arbitrary subvolume (0 in this case)
+     * as parent read subvol.
+     */
+    if (par_read_subvol_iter == -1)
+        par_read_subvol_iter = 0;
+
+    return par_read_subvol_iter;
+}
 
 int
-afr_flush_unwind (call_frame_t *frame, xlator_t *this)
+afr_read_subvol_decide(inode_t *inode, xlator_t *this,
+                       afr_read_subvol_args_t *args, unsigned char *readable)
 {
-        afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
+    int event = 0;
+    afr_private_t *priv = NULL;
+    unsigned char *intersection = NULL;
 
-        local = frame->local;
+    priv = this->private;
+    intersection = alloca0(priv->child_count);
 
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame)
-                        main_frame = local->transaction.main_frame;
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
+    afr_readables_intersect_get(inode, this, &event, intersection);
 
-        if (main_frame) {
-                AFR_STACK_UNWIND (flush, main_frame,
-                                  local->op_ret, local->op_errno);
-        }
+    if (AFR_COUNT(intersection, priv->child_count) <= 0) {
+        /* TODO: If we have one brick with valid data_readable and
+         * another with metadata_readable, try to send an iatt with
+         * valid bits from both.*/
+        return -1;
+    }
 
-        return 0;
+    memcpy(readable, intersection, sizeof(*readable) * priv->child_count);
+
+    return afr_read_subvol_select_by_policy(inode, this, intersection, args);
 }
 
+static inline int
+afr_first_up_child(call_frame_t *frame, xlator_t *this)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int i = 0;
+
+    local = frame->local;
+    priv = this->private;
 
-int
-afr_flush_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno)
+    for (i = 0; i < priv->child_count; i++)
+        if (local->replies[i].valid && local->replies[i].op_ret == 0)
+            return i;
+    return -1;
+}
+
+static void
+afr_attempt_readsubvol_set(call_frame_t *frame, xlator_t *this,
+                           unsigned char *success_replies,
+                           unsigned char *data_readable, int *read_subvol)
 {
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int spb_subvol = -1;
+    int child_count = -1;
 
-        int call_count  = -1;
-        int child_index = (long) cookie;
-        int need_unwind = 0;
+    if (*read_subvol != -1)
+        return;
 
-        local = frame->local;
-        priv  = this->private;
+    priv = this->private;
+    local = frame->local;
+    child_count = priv->child_count;
+
+    afr_split_brain_read_subvol_get(local->inode, this, frame, &spb_subvol);
+    if ((spb_subvol >= 0) &&
+        (AFR_COUNT(success_replies, child_count) == child_count)) {
+        *read_subvol = spb_subvol;
+    } else if (!priv->quorum_count ||
+               frame->root->pid == GF_CLIENT_PID_GLFS_HEAL) {
+        *read_subvol = afr_first_up_child(frame, this);
+    } else if (priv->quorum_count &&
+               afr_has_quorum(data_readable, this, NULL)) {
+        /* read_subvol is guaranteed to be valid if we hit this path. */
+        *read_subvol = afr_first_up_child(frame, this);
+    } else {
+        /* If quorum is enabled and we do not have a
+           readable yet, it means all good copies are down.
+        */
+        local->op_ret = -1;
+        local->op_errno = ENOTCONN;
+        gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_READ_SUBVOL_ERROR,
+               "no read "
+               "subvols for %s",
+               local->loc.path);
+    }
+    if (*read_subvol >= 0)
+        dict_del_sizen(local->replies[*read_subvol].xdata, GF_CONTENT_KEY);
+}
 
-        LOCK (&frame->lock);
-        {
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
-
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                        }
-                        local->success_count++;
-
-                        if (local->success_count == priv->wait_count) {
-                                need_unwind = 1;
-                        }
-                }
+static void
+afr_lookup_done(call_frame_t *frame, xlator_t *this)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int i = -1;
+    int op_errno = 0;
+    int read_subvol = 0;
+    int par_read_subvol = 0;
+    int ret = -1;
+    unsigned char *readable = NULL;
+    unsigned char *success_replies = NULL;
+    int event = 0;
+    struct afr_reply *replies = NULL;
+    uuid_t read_gfid = {
+        0,
+    };
+    gf_boolean_t locked_entry = _gf_false;
+    gf_boolean_t in_flight_create = _gf_false;
+    gf_boolean_t can_interpret = _gf_true;
+    inode_t *parent = NULL;
+    ia_type_t ia_type = IA_INVAL;
+    afr_read_subvol_args_t args = {
+        0,
+    };
+    char *gfid_heal_msg = NULL;
+
+    priv = this->private;
+    local = frame->local;
+    replies = local->replies;
+    parent = local->loc.parent;
+
+    locked_entry = afr_is_possibly_under_txn(AFR_ENTRY_TRANSACTION, local,
+                                             this);
+
+    readable = alloca0(priv->child_count);
+    success_replies = alloca0(priv->child_count);
+
+    afr_inode_read_subvol_get(parent, this, readable, NULL, &event);
+    par_read_subvol = afr_get_parent_read_subvol(this, parent, replies,
+                                                 readable);
+
+    /* First, check if we have a gfid-change from somewhere,
+       If so, propagate that so that a fresh lookup can be
+       issued
+    */
+    if (local->cont.lookup.needs_fresh_lookup) {
+        local->op_ret = -1;
+        local->op_errno = ESTALE;
+        goto error;
+    }
 
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
+    op_errno = afr_final_errno(frame->local, this->private);
+    local->op_errno = op_errno;
 
-        if (need_unwind)
-                afr_flush_unwind (frame, this);
+    read_subvol = -1;
+    afr_fill_success_replies(local, priv, success_replies);
 
-        call_count = afr_frame_return (frame);
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid)
+            continue;
 
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
+        if (replies[i].op_ret == -1) {
+            if (locked_entry && replies[i].op_errno == ENOENT) {
+                in_flight_create = _gf_true;
+            }
+            continue;
         }
 
-        return 0;
-}
+        if (read_subvol == -1 || !readable[read_subvol]) {
+            read_subvol = i;
+            gf_uuid_copy(read_gfid, replies[i].poststat.ia_gfid);
+            ia_type = replies[i].poststat.ia_type;
+            local->op_ret = 0;
+        }
+    }
 
+    if (in_flight_create && !afr_has_quorum(success_replies, this, NULL)) {
+        local->op_ret = -1;
+        local->op_errno = ENOENT;
+        goto error;
+    }
+
+    if (read_subvol == -1)
+        goto error;
+    /* We now have a read_subvol, which is readable[] (if there
+       were any). Next we look for GFID mismatches. We don't
+       consider a GFID mismatch as an error if read_subvol is
+       readable[] but the mismatching GFID subvol is not.
+    */
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret == -1) {
+            continue;
+        }
 
-int
-afr_flush_wind (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+        if (!gf_uuid_compare(replies[i].poststat.ia_gfid, read_gfid))
+            continue;
 
-        int i = 0;
-        int call_count = -1;
+        can_interpret = _gf_false;
 
-        local = frame->local;
-        priv = this->private;
+        if (locked_entry)
+            continue;
+
+        /* Now GFIDs mismatch. It's OK as long as this subvol
+           is not readable[] but read_subvol is */
+        if (readable[read_subvol] && !readable[i])
+            continue;
 
-        call_count = afr_up_children_count (priv->child_count, local->child_up);
+        /* If we were called from glfsheal and there is still a gfid
+         * mismatch, succeed the lookup and let glfsheal print the
+         * response via gfid-heal-msg.*/
+        if (!dict_get_str_sizen(local->xattr_req, "gfid-heal-msg",
+                                &gfid_heal_msg))
+            goto cant_interpret;
 
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
+        /* LOG ERROR */
+        local->op_ret = -1;
+        local->op_errno = EIO;
+        goto error;
+    }
+
+    /* Forth, for the finalized GFID, pick the best subvolume
+       to return stats from.
+    */
+    read_subvol = -1;
+    memset(readable, 0, sizeof(*readable) * priv->child_count);
+    if (can_interpret) {
+        if (!afr_has_quorum(success_replies, this, NULL))
+            goto cant_interpret;
+        /* It is safe to call afr_replies_interpret() because we have
+           a response from all the UP subvolumes and all of them resolved
+           to the same GFID
+        */
+        gf_uuid_copy(args.gfid, read_gfid);
+        args.ia_type = ia_type;
+        ret = afr_replies_interpret(frame, this, local->inode, NULL);
+        read_subvol = afr_read_subvol_decide(local->inode, this, &args,
+                                             readable);
+        if (read_subvol == -1)
+            goto cant_interpret;
+        if (ret) {
+            afr_inode_need_refresh_set(local->inode, this);
+            dict_del_sizen(local->replies[read_subvol].xdata, GF_CONTENT_KEY);
+        }
+    } else {
+    cant_interpret:
+        afr_attempt_readsubvol_set(frame, this, success_replies, readable,
+                                   &read_subvol);
+        if (read_subvol == -1) {
+            goto error;
         }
+    }
 
-        local->call_count = call_count;
+    afr_handle_quota_size(frame, this);
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, afr_flush_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->flush,
-                                           local->fd);
-
-                        if (!--call_count)
-                                break;
-                }
+    afr_set_need_heal(this, local);
+    if (AFR_IS_ARBITER_BRICK(priv, read_subvol) && local->op_ret == 0) {
+        local->op_ret = -1;
+        local->op_errno = ENOTCONN;
+        gf_msg_debug(this->name, 0,
+                     "Arbiter cannot be a read subvol "
+                     "for %s",
+                     local->loc.path);
+        goto error;
+    }
+
+    ret = dict_get_str_sizen(local->xattr_req, "gfid-heal-msg", &gfid_heal_msg);
+    if (!ret) {
+        ret = dict_set_str_sizen(local->replies[read_subvol].xdata,
+                                 "gfid-heal-msg", gfid_heal_msg);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_DICT_SET_FAILED,
+                   "Error setting gfid-heal-msg dict");
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
         }
+    }
 
-        return 0;
+    AFR_STACK_UNWIND(lookup, frame, local->op_ret, local->op_errno,
+                     local->inode, &local->replies[read_subvol].poststat,
+                     local->replies[read_subvol].xdata,
+                     &local->replies[par_read_subvol].postparent);
+    return;
+
+error:
+    AFR_STACK_UNWIND(lookup, frame, local->op_ret, local->op_errno, NULL, NULL,
+                     NULL, NULL);
 }
 
+/*
+ * During a lookup, some errors are more "important" than
+ * others in that they must be given higher priority while
+ * returning to the user.
+ *
+ * The hierarchy is ENODATA > ENOENT > ESTALE > ENOSPC others
+ */
 
 int
-afr_flush_done (call_frame_t *frame, xlator_t *this)
+afr_higher_errno(int32_t old_errno, int32_t new_errno)
 {
-        afr_local_t *local = NULL;
+    if (old_errno == ENODATA || new_errno == ENODATA)
+        return ENODATA;
+    if (old_errno == ENOENT || new_errno == ENOENT)
+        return ENOENT;
+    if (old_errno == ESTALE || new_errno == ESTALE)
+        return ESTALE;
+    if (old_errno == ENOSPC || new_errno == ENOSPC)
+        return ENOSPC;
+
+    return new_errno;
+}
 
-        local = frame->local;
+int
+afr_final_errno(afr_local_t *local, afr_private_t *priv)
+{
+    int i = 0;
+    int op_errno = 0;
+    int tmp_errno = 0;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid)
+            continue;
+        if (local->replies[i].op_ret >= 0)
+            continue;
+        tmp_errno = local->replies[i].op_errno;
+        op_errno = afr_higher_errno(op_errno, tmp_errno);
+    }
+
+    return op_errno;
+}
 
-        local->transaction.unwind (frame, this);
+static int32_t
+afr_local_discovery_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *dict,
+                        dict_t *xdata)
+{
+    int ret = 0;
+    char *pathinfo = NULL;
+    gf_boolean_t is_local = _gf_false;
+    afr_private_t *priv = NULL;
+    int32_t child_index = -1;
+
+    if (op_ret != 0) {
+        goto out;
+    }
+
+    priv = this->private;
+    child_index = (int32_t)(long)cookie;
+
+    ret = dict_get_str_sizen(dict, GF_XATTR_PATHINFO_KEY, &pathinfo);
+    if (ret != 0) {
+        goto out;
+    }
+
+    ret = glusterfs_is_local_pathinfo(pathinfo, &is_local);
+    if (ret) {
+        goto out;
+    }
+
+    /*
+     * Note that one local subvolume will override another here.  The only
+     * way to avoid that would be to retain extra information about whether
+     * the previous read_child is local, and it's just not worth it.  Even
+     * the slowest local subvolume is far preferable to a remote one.
+     */
+    if (is_local) {
+        priv->local[child_index] = 1;
+        /* Don't set arbiter as read child. */
+        if (AFR_IS_ARBITER_BRICK(priv, child_index))
+            goto out;
+        gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_LOCAL_CHILD,
+               "selecting local read_child %s",
+               priv->children[child_index]->name);
+
+        priv->read_child = child_index;
+    }
+out:
+    STACK_DESTROY(frame->root);
+    return 0;
+}
 
-        AFR_STACK_DESTROY (frame);
+static void
+afr_attempt_local_discovery(xlator_t *this, int32_t child_index)
+{
+    call_frame_t *newframe = NULL;
+    loc_t tmploc = {
+        0,
+    };
+    afr_private_t *priv = this->private;
+
+    newframe = create_frame(this, this->ctx->pool);
+    if (!newframe) {
+        return;
+    }
 
-        return 0;
+    tmploc.gfid[sizeof(tmploc.gfid) - 1] = 1;
+    STACK_WIND_COOKIE(newframe, afr_local_discovery_cbk,
+                      (void *)(long)child_index, priv->children[child_index],
+                      priv->children[child_index]->fops->getxattr, &tmploc,
+                      GF_XATTR_PATHINFO_KEY, NULL);
 }
 
-
 int
-afr_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
+afr_lookup_sh_metadata_wrap(void *opaque)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
+    call_frame_t *frame = opaque;
+    afr_local_t *local = NULL;
+    xlator_t *this = NULL;
+    inode_t *inode = NULL;
+    afr_private_t *priv = NULL;
+    struct afr_reply *replies = NULL;
+    int i = 0, first = -1;
+    int ret = -1;
+    dict_t *dict = NULL;
+
+    local = frame->local;
+    this = frame->this;
+    priv = this->private;
+    replies = local->replies;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret == -1)
+            continue;
+        first = i;
+        break;
+    }
+    if (first == -1)
+        goto out;
+
+    if (afr_selfheal_metadata_by_stbuf(this, &replies[first].poststat))
+        goto out;
+
+    afr_local_replies_wipe(local, this->private);
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+    if (local->xattr_req) {
+        dict_copy(local->xattr_req, dict);
+    }
+
+    ret = dict_set_sizen_str_sizen(dict, "link-count", GF_XATTROP_INDEX_COUNT);
+    if (ret) {
+        gf_msg_debug(this->name, -ret, "Unable to set link-count in dict ");
+    }
+
+    if (loc_is_nameless(&local->loc)) {
+        ret = afr_selfheal_unlocked_discover_on(frame, local->inode,
+                                                local->loc.gfid, local->replies,
+                                                local->child_up, dict);
+    } else {
+        inode = afr_selfheal_unlocked_lookup_on(frame, local->loc.parent,
+                                                local->loc.name, local->replies,
+                                                local->child_up, dict);
+    }
+    if (inode)
+        inode_unref(inode);
+out:
+    if (loc_is_nameless(&local->loc))
+        afr_discover_done(frame, this);
+    else
+        afr_lookup_done(frame, this);
 
-        call_frame_t  * transaction_frame = NULL;
+    if (dict)
+        dict_unref(dict);
 
-        int ret        = -1;
+    return 0;
+}
 
-        int op_ret   = -1;
-        int op_errno = 0;
+gf_boolean_t
+afr_is_pending_set(xlator_t *this, dict_t *xdata, int type)
+{
+    int idx = -1;
+    afr_private_t *priv = NULL;
+    void *pending_raw = NULL;
+    int *pending_int = NULL;
+    int i = 0;
 
-        int call_count = 0;
+    priv = this->private;
+    idx = afr_index_for_transaction_type(type);
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+    if (dict_get_ptr(xdata, AFR_DIRTY, &pending_raw) == 0) {
+        if (pending_raw) {
+            pending_int = pending_raw;
 
-        priv = this->private;
+            if (ntoh32(pending_int[idx]))
+                return _gf_true;
+        }
+    }
 
-        ALLOC_OR_GOTO (local, afr_local_t, out);
+    for (i = 0; i < priv->child_count; i++) {
+        if (dict_get_ptr(xdata, priv->pending_key[i], &pending_raw))
+            continue;
+        if (!pending_raw)
+            continue;
+        pending_int = pending_raw;
 
-        ret = AFR_LOCAL_INIT (local, priv);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
+        if (ntoh32(pending_int[idx]))
+            return _gf_true;
+    }
+
+    return _gf_false;
+}
+
+static gf_boolean_t
+afr_can_start_metadata_self_heal(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    struct afr_reply *replies = NULL;
+    int i = 0, first = -1;
+    gf_boolean_t start = _gf_false;
+    struct iatt stbuf = {
+        0,
+    };
+
+    local = frame->local;
+    replies = local->replies;
+    priv = this->private;
+
+    if (!priv->metadata_self_heal)
+        return _gf_false;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret == -1)
+            continue;
+        if (first == -1) {
+            first = i;
+            stbuf = replies[i].poststat;
+            continue;
         }
 
-        call_count = afr_up_children_count (priv->child_count, local->child_up);
+        if (afr_is_pending_set(this, replies[i].xdata,
+                               AFR_METADATA_TRANSACTION)) {
+            /* Let shd do the heal so that lookup is not blocked
+             * on getting metadata lock/doing the heal */
+            start = _gf_false;
+            break;
+        }
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory.");
-                goto out;
+        if (gf_uuid_compare(stbuf.ia_gfid, replies[i].poststat.ia_gfid)) {
+            start = _gf_false;
+            break;
+        }
+        if (!IA_EQUAL(stbuf, replies[i].poststat, type)) {
+            start = _gf_false;
+            break;
         }
 
-        transaction_frame->local = local;
+        /*Check if iattrs need heal*/
+        if ((!IA_EQUAL(stbuf, replies[i].poststat, uid)) ||
+            (!IA_EQUAL(stbuf, replies[i].poststat, gid)) ||
+            (!IA_EQUAL(stbuf, replies[i].poststat, prot))) {
+            start = _gf_true;
+            continue;
+        }
 
-        local->op = GF_FOP_FLUSH;
+        /*Check if xattrs need heal*/
+        if (!afr_xattrs_are_equal(replies[first].xdata, replies[i].xdata))
+            start = _gf_true;
+    }
 
-        local->transaction.fop    = afr_flush_wind;
-        local->transaction.done   = afr_flush_done;
-        local->transaction.unwind = afr_flush_unwind;
+    return start;
+}
 
-        local->fd                 = fd_ref (fd);
+int
+afr_lookup_metadata_heal_check(call_frame_t *frame, xlator_t *this)
 
-        local->transaction.main_frame = frame;
-        local->transaction.start  = 0;
-        local->transaction.len    = 0;
+{
+    call_frame_t *heal = NULL;
+    afr_local_t *local = NULL;
+    int ret = 0;
+
+    local = frame->local;
+    if (!afr_can_start_metadata_self_heal(frame, this))
+        goto out;
+
+    heal = afr_frame_create(this, &ret);
+    if (!heal) {
+        ret = -ret;
+        goto out;
+    }
+
+    ret = synctask_new(this->ctx->env, afr_lookup_sh_metadata_wrap,
+                       afr_refresh_selfheal_done, heal, frame);
+    if (ret)
+        goto out;
+    return ret;
+out:
+    if (loc_is_nameless(&local->loc))
+        afr_discover_done(frame, this);
+    else
+        afr_lookup_done(frame, this);
+    if (heal)
+        AFR_STACK_DESTROY(heal);
+    return ret;
+}
 
-        afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
+int
+afr_lookup_selfheal_wrap(void *opaque)
+{
+    int ret = 0;
+    call_frame_t *frame = opaque;
+    afr_local_t *local = NULL;
+    xlator_t *this = NULL;
+    inode_t *inode = NULL;
+    uuid_t pargfid = {
+        0,
+    };
+
+    local = frame->local;
+    this = frame->this;
+    loc_pargfid(&local->loc, pargfid);
+
+    ret = afr_selfheal_name(frame->this, pargfid, local->loc.name,
+                            &local->cont.lookup.gfid_req, local->xattr_req);
+    if (ret == -EIO)
+        goto unwind;
+
+    afr_local_replies_wipe(local, this->private);
+
+    inode = afr_selfheal_unlocked_lookup_on(frame, local->loc.parent,
+                                            local->loc.name, local->replies,
+                                            local->child_up, local->xattr_req);
+    if (inode)
+        inode_unref(inode);
+
+    afr_lookup_metadata_heal_check(frame, this);
+    return 0;
 
+unwind:
+    AFR_STACK_UNWIND(lookup, frame, -1, EIO, NULL, NULL, NULL, NULL);
+    return 0;
+}
 
-        op_ret = 0;
-out:
-        if (op_ret == -1) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
+int
+afr_lookup_entry_heal(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    call_frame_t *heal = NULL;
+    int i = 0, first = -1;
+    gf_boolean_t name_state_mismatch = _gf_false;
+    struct afr_reply *replies = NULL;
+    int ret = 0;
+    unsigned char *par_readables = NULL;
+    unsigned char *success = NULL;
+    int32_t op_errno = 0;
+    uuid_t gfid = {0};
+
+    local = frame->local;
+    replies = local->replies;
+    priv = this->private;
+    par_readables = alloca0(priv->child_count);
+    success = alloca0(priv->child_count);
+
+    ret = afr_inode_read_subvol_get(local->loc.parent, this, par_readables,
+                                    NULL, NULL);
+    if (ret < 0 || AFR_COUNT(par_readables, priv->child_count) == 0) {
+        /* In this case set par_readables to all 1 so that name_heal
+         * need checks at the end of this function will flag missing
+         * entry when name state mismatches*/
+        memset(par_readables, 1, priv->child_count);
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid)
+            continue;
+
+        if (replies[i].op_ret == 0) {
+            if (gf_uuid_is_null(gfid)) {
+                gf_uuid_copy(gfid, replies[i].poststat.ia_gfid);
+            }
+            success[i] = 1;
+        } else {
+            if ((replies[i].op_errno != ENOTCONN) &&
+                (replies[i].op_errno != ENOENT) &&
+                (replies[i].op_errno != ESTALE)) {
+                op_errno = replies[i].op_errno;
+            }
+        }
 
-                AFR_STACK_UNWIND (flush, frame, op_ret, op_errno);
+        /*gfid is missing, needs heal*/
+        if ((replies[i].op_ret == -1) && (replies[i].op_errno == ENODATA)) {
+            goto name_heal;
         }
 
-        return 0;
-}
+        if (first == -1) {
+            first = i;
+            continue;
+        }
 
-/* }}} */
+        if (replies[i].op_ret != replies[first].op_ret) {
+            name_state_mismatch = _gf_true;
+        }
+
+        if (replies[i].op_ret == 0) {
+            /* Rename after this lookup may succeed if we don't do
+             * a name-heal and the destination may not have pending xattrs
+             * to indicate which name is good and which is bad so always do
+             * this heal*/
+            if (gf_uuid_compare(replies[i].poststat.ia_gfid, gfid)) {
+                goto name_heal;
+            }
+        }
+    }
+
+    if (name_state_mismatch) {
+        if (!priv->quorum_count)
+            goto name_heal;
+        if (!afr_has_quorum(success, this, NULL))
+            goto name_heal;
+        if (op_errno)
+            goto name_heal;
+        for (i = 0; i < priv->child_count; i++) {
+            if (!replies[i].valid)
+                continue;
+            if (par_readables[i] && replies[i].op_ret < 0 &&
+                replies[i].op_errno != ENOTCONN) {
+                goto name_heal;
+            }
+        }
+    }
+
+    goto metadata_heal;
 
+name_heal:
+    heal = afr_frame_create(this, NULL);
+    if (!heal)
+        goto metadata_heal;
+
+    ret = synctask_new(this->ctx->env, afr_lookup_selfheal_wrap,
+                       afr_refresh_selfheal_done, heal, frame);
+    if (ret) {
+        AFR_STACK_DESTROY(heal);
+        goto metadata_heal;
+    }
+    return ret;
+
+metadata_heal:
+    ret = afr_lookup_metadata_heal_check(frame, this);
+
+    return ret;
+}
 
 int
-afr_cleanup_fd_ctx (xlator_t *this, fd_t *fd)
+afr_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, inode_t *inode, struct iatt *buf, dict_t *xdata,
+               struct iatt *postparent)
 {
-        uint64_t        ctx = 0;
-        afr_fd_ctx_t    *fd_ctx = NULL;
-        int             ret = 0;
-
-        ret = fd_ctx_get (fd, this, &ctx);
+    afr_local_t *local = NULL;
+    int call_count = -1;
+    int child_index = -1;
+    GF_UNUSED int ret = 0;
+    int8_t need_heal = 1;
+
+    child_index = (long)cookie;
+
+    local = frame->local;
+
+    local->replies[child_index].valid = 1;
+    local->replies[child_index].op_ret = op_ret;
+    local->replies[child_index].op_errno = op_errno;
+    /*
+     * On revalidate lookup if the gfid-changed, afr should unwind the fop
+     * with ESTALE so that a fresh lookup will be sent by the top xlator.
+     * So remember it.
+     */
+    if (xdata && dict_get_sizen(xdata, "gfid-changed"))
+        local->cont.lookup.needs_fresh_lookup = _gf_true;
+
+    if (xdata) {
+        ret = dict_get_int8(xdata, "link-count", &need_heal);
+        local->replies[child_index].need_heal = need_heal;
+    } else {
+        local->replies[child_index].need_heal = need_heal;
+    }
+    if (op_ret != -1) {
+        local->replies[child_index].poststat = *buf;
+        local->replies[child_index].postparent = *postparent;
+        if (xdata)
+            local->replies[child_index].xdata = dict_ref(xdata);
+    }
+
+    call_count = afr_frame_return(frame);
+    if (call_count == 0) {
+        afr_set_need_heal(this, local);
+        afr_lookup_entry_heal(frame, this);
+    }
+
+    return 0;
+}
 
-        if (ret < 0)
-                goto out;
+static void
+afr_discover_unwind(call_frame_t *frame, xlator_t *this)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int read_subvol = -1;
+    int ret = 0;
+    unsigned char *data_readable = NULL;
+    unsigned char *success_replies = NULL;
+
+    priv = this->private;
+    local = frame->local;
+    data_readable = alloca0(priv->child_count);
+    success_replies = alloca0(priv->child_count);
+
+    afr_fill_success_replies(local, priv, success_replies);
+    if (AFR_COUNT(success_replies, priv->child_count) > 0)
+        local->op_ret = 0;
+
+    if (local->op_ret < 0) {
+        local->op_ret = -1;
+        local->op_errno = afr_final_errno(frame->local, this->private);
+        goto error;
+    }
 
-        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
+    if (!afr_has_quorum(success_replies, this, frame))
+        goto unwind;
 
-        if (fd_ctx) {
-                if (fd_ctx->pre_op_done)
-                        GF_FREE (fd_ctx->pre_op_done);
+    ret = afr_replies_interpret(frame, this, local->inode, NULL);
+    if (ret) {
+        afr_inode_need_refresh_set(local->inode, this);
+    }
 
-                if (fd_ctx->opened_on)
-                        GF_FREE (fd_ctx->opened_on);
+    read_subvol = afr_read_subvol_decide(local->inode, this, NULL,
+                                         data_readable);
 
-                if (fd_ctx->locked_on)
-                        GF_FREE (fd_ctx->locked_on);
+unwind:
+    afr_attempt_readsubvol_set(frame, this, success_replies, data_readable,
+                               &read_subvol);
+    if (read_subvol == -1)
+        goto error;
 
-                if (fd_ctx->pre_op_piggyback)
-                        GF_FREE (fd_ctx->pre_op_piggyback);
+    if (AFR_IS_ARBITER_BRICK(priv, read_subvol) && local->op_ret == 0) {
+        local->op_ret = -1;
+        local->op_errno = ENOTCONN;
+        gf_msg_debug(this->name, 0,
+                     "Arbiter cannot be a read subvol "
+                     "for %s",
+                     local->loc.path);
+    }
+
+    AFR_STACK_UNWIND(lookup, frame, local->op_ret, local->op_errno,
+                     local->inode, &local->replies[read_subvol].poststat,
+                     local->replies[read_subvol].xdata,
+                     &local->replies[read_subvol].postparent);
+    return;
+
+error:
+    AFR_STACK_UNWIND(lookup, frame, local->op_ret, local->op_errno, NULL, NULL,
+                     NULL, NULL);
+}
 
-                GF_FREE (fd_ctx);
-        }
+static int
+afr_ta_id_file_check(void *opaque)
+{
+    afr_private_t *priv = NULL;
+    xlator_t *this = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt stbuf = {
+        0,
+    };
+    dict_t *dict = NULL;
+    uuid_t gfid = {
+        0,
+    };
+    fd_t *fd = NULL;
+    int ret = 0;
+
+    this = opaque;
+    priv = this->private;
+
+    ret = afr_fill_ta_loc(this, &loc, _gf_false);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed to populate thin-arbiter loc for: %s.", loc.name);
+        goto out;
+    }
+
+    ret = syncop_lookup(priv->children[THIN_ARBITER_BRICK_INDEX], &loc, &stbuf,
+                        0, 0, 0);
+    if (ret == 0) {
+        goto out;
+    } else if (ret == -ENOENT) {
+        fd = fd_create(loc.inode, getpid());
+        if (!fd)
+            goto out;
+        dict = dict_new();
+        if (!dict)
+            goto out;
+        gf_uuid_generate(gfid);
+        ret = dict_set_gfuuid(dict, "gfid-req", gfid, true);
+        ret = syncop_create(priv->children[THIN_ARBITER_BRICK_INDEX], &loc,
+                            O_RDWR, 0664, fd, &stbuf, dict, NULL);
+    }
 
 out:
-        return 0;
+    if (ret == 0) {
+        gf_uuid_copy(priv->ta_gfid, stbuf.ia_gfid);
+    } else {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed to lookup/create thin-arbiter id file.");
+    }
+    if (dict)
+        dict_unref(dict);
+    if (fd)
+        fd_unref(fd);
+    loc_wipe(&loc);
+
+    return 0;
 }
 
+static int
+afr_ta_id_file_check_cbk(int ret, call_frame_t *ta_frame, void *opaque)
+{
+    return 0;
+}
+
+static void
+afr_discover_done(call_frame_t *frame, xlator_t *this)
+{
+    int ret = 0;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+    if (!priv->thin_arbiter_count)
+        goto unwind;
+    if (!gf_uuid_is_null(priv->ta_gfid))
+        goto unwind;
+
+    ret = synctask_new(this->ctx->env, afr_ta_id_file_check,
+                       afr_ta_id_file_check_cbk, NULL, this);
+    if (ret)
+        goto unwind;
+unwind:
+    afr_discover_unwind(frame, this);
+}
 
 int
-afr_release (xlator_t *this, fd_t *fd)
+afr_discover_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, inode_t *inode, struct iatt *buf, dict_t *xdata,
+                 struct iatt *postparent)
 {
-        afr_locked_fd_t *locked_fd = NULL;
-        afr_locked_fd_t *tmp       = NULL;
-        afr_private_t   *priv      = NULL;
+    afr_local_t *local = NULL;
+    int call_count = -1;
+    int child_index = -1;
+    GF_UNUSED int ret = 0;
+    int8_t need_heal = 1;
+
+    child_index = (long)cookie;
+
+    local = frame->local;
+
+    local->replies[child_index].valid = 1;
+    local->replies[child_index].op_ret = op_ret;
+    local->replies[child_index].op_errno = op_errno;
+    if (op_ret != -1) {
+        local->replies[child_index].poststat = *buf;
+        local->replies[child_index].postparent = *postparent;
+        if (xdata)
+            local->replies[child_index].xdata = dict_ref(xdata);
+    }
+
+    if (local->do_discovery && (op_ret == 0))
+        afr_attempt_local_discovery(this, child_index);
+
+    if (xdata) {
+        ret = dict_get_int8(xdata, "link-count", &need_heal);
+        local->replies[child_index].need_heal = need_heal;
+    } else {
+        local->replies[child_index].need_heal = need_heal;
+    }
+
+    call_count = afr_frame_return(frame);
+    if (call_count == 0) {
+        afr_set_need_heal(this, local);
+        afr_lookup_metadata_heal_check(frame, this);
+    }
+
+    return 0;
+}
 
-        priv = this->private;
+int
+afr_discover_do(call_frame_t *frame, xlator_t *this, int err)
+{
+    int ret = 0;
+    int i = 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int call_count = 0;
+
+    local = frame->local;
+    priv = this->private;
+
+    if (err) {
+        local->op_errno = err;
+        goto out;
+    }
+
+    call_count = local->call_count = AFR_COUNT(local->child_up,
+                                               priv->child_count);
+
+    ret = afr_lookup_xattr_req_prepare(local, this, local->xattr_req,
+                                       &local->loc);
+    if (ret) {
+        local->op_errno = -ret;
+        goto out;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->child_up[i]) {
+            STACK_WIND_COOKIE(
+                frame, afr_discover_cbk, (void *)(long)i, priv->children[i],
+                priv->children[i]->fops->lookup, &local->loc, local->xattr_req);
+            if (!--call_count)
+                break;
+        }
+    }
+
+    return 0;
+out:
+    AFR_STACK_UNWIND(lookup, frame, -1, local->op_errno, 0, 0, 0, 0);
+    return 0;
+}
 
-        afr_cleanup_fd_ctx (this, fd);
+int
+afr_discover(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr_req)
+{
+    int op_errno = ENOMEM;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int event = 0;
+
+    priv = this->private;
+
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
+
+    if (!local->call_count) {
+        op_errno = ENOTCONN;
+        goto out;
+    }
+
+    if (__is_root_gfid(loc->inode->gfid)) {
+        if (!priv->root_inode)
+            priv->root_inode = inode_ref(loc->inode);
+
+        if (priv->choose_local && !priv->did_discovery) {
+            /* Logic to detect which subvolumes of AFR are
+               local, in order to prefer them for reads
+            */
+            local->do_discovery = _gf_true;
+            priv->did_discovery = _gf_true;
+        }
+    }
 
-        list_for_each_entry_safe (locked_fd, tmp, &priv->saved_fds,
-                                  list) {
+    local->op = GF_FOP_LOOKUP;
 
-                if (locked_fd->fd == fd) {
-                        list_del_init (&locked_fd->list);
-                        GF_FREE (locked_fd);
-                }
+    loc_copy(&local->loc, loc);
 
+    local->inode = inode_ref(loc->inode);
+
+    if (xattr_req) {
+        /* If xattr_req was null, afr_lookup_xattr_req_prepare() will
+           allocate one for us */
+        local->xattr_req = dict_copy_with_ref(xattr_req, NULL);
+        if (!local->xattr_req) {
+            op_errno = ENOMEM;
+            goto out;
         }
+    }
 
+    if (gf_uuid_is_null(loc->inode->gfid)) {
+        afr_discover_do(frame, this, 0);
         return 0;
-}
+    }
+
+    afr_read_subvol_get(loc->inode, this, NULL, NULL, &event,
+                        AFR_DATA_TRANSACTION, NULL);
 
+    afr_discover_do(frame, this, 0);
 
-/* {{{ fsync */
+    return 0;
+out:
+    AFR_STACK_UNWIND(lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+    return 0;
+}
 
 int
-afr_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-               struct iatt *postbuf)
+afr_lookup_do(call_frame_t *frame, xlator_t *this, int err)
 {
-        afr_local_t *local = NULL;
-
-        int call_count = -1;
+    int ret = 0;
+    int i = 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int call_count = 0;
+
+    local = frame->local;
+    priv = this->private;
+
+    if (err < 0) {
+        local->op_errno = err;
+        goto out;
+    }
+
+    call_count = local->call_count = AFR_COUNT(local->child_up,
+                                               priv->child_count);
+
+    ret = afr_lookup_xattr_req_prepare(local, this, local->xattr_req,
+                                       &local->loc);
+    if (ret) {
+        local->op_errno = -ret;
+        goto out;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->child_up[i]) {
+            STACK_WIND_COOKIE(
+                frame, afr_lookup_cbk, (void *)(long)i, priv->children[i],
+                priv->children[i]->fops->lookup, &local->loc, local->xattr_req);
+            if (!--call_count)
+                break;
+        }
+    }
+    return 0;
+out:
+    AFR_STACK_UNWIND(lookup, frame, -1, local->op_errno, 0, 0, 0, 0);
+    return 0;
+}
 
-        int child_index = (long) cookie;
-        int read_child  = 0;
+/*
+ * afr_lookup()
+ *
+ * The goal here is to figure out what the element getting looked up is.
+ * i.e what is the GFID, inode type and a conservative estimate of the
+ * inode attributes are.
+ *
+ * As we lookup, operations may be underway on the entry name and the
+ * inode. In lookup() we are primarily concerned only with the entry
+ * operations. If the entry is getting unlinked or renamed, we detect
+ * what operation is underway by querying for on-going transactions and
+ * pending self-healing on the entry through xdata.
+ *
+ * If the entry is a file/dir, it may need self-heal and/or in a
+ * split-brain condition. Lookup is not the place to worry about these
+ * conditions. Outcast marking will naturally handle them in the read
+ * paths.
+ *
+ * Here is a brief goal of what we are trying to achieve:
+ *
+ * - LOOKUP on all subvolumes concurrently, querying on-going transaction
+ *   and pending self-heal info from the servers.
+ *
+ * - If all servers reply the same inode type and GFID, the overall call
+ *   MUST be a success.
+ *
+ * - If inode types or GFIDs mismatch, and there IS either an on-going
+ *   transaction or pending self-heal, inspect what the nature of the
+ *   transaction or pending heal is, and select the appropriate subvolume's
+ *   reply as the winner.
+ *
+ * - If inode types or GFIDs mismatch, and there are no on-going transactions
+ *   or pending self-heal on the entry name on any of the servers, fail the
+ *   lookup with EIO. Something has gone wrong beyond reasonable action.
+ */
 
-        local = frame->local;
+int
+afr_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr_req)
+{
+    afr_local_t *local = NULL;
+    int32_t op_errno = 0;
+    int event = 0;
+    int ret = 0;
+
+    if (loc_is_nameless(loc)) {
+        if (xattr_req)
+            dict_del_sizen(xattr_req, "gfid-req");
+        afr_discover(frame, this, loc, xattr_req);
+        return 0;
+    }
 
-        read_child = afr_read_child (this, local->fd->inode);
+    if (afr_is_private_directory(this->private, loc->parent->gfid, loc->name,
+                                 frame->root->pid)) {
+        op_errno = EPERM;
+        goto out;
+    }
 
-        LOCK (&frame->lock);
-        {
-                if (child_index == read_child) {
-                        local->read_child_returned = _gf_true;
-                }
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-                if (op_ret == 0) {
-                        local->op_ret = 0;
+    if (!local->call_count) {
+        op_errno = ENOTCONN;
+        goto out;
+    }
 
-                        if (local->success_count == 0) {
-                                local->cont.fsync.prebuf  = *prebuf;
-                                local->cont.fsync.postbuf = *postbuf;
-                        }
+    local->op = GF_FOP_LOOKUP;
 
-                        if (child_index == read_child) {
-                                local->cont.fsync.prebuf  = *prebuf;
-                                local->cont.fsync.postbuf = *postbuf;
-                        }
+    loc_copy(&local->loc, loc);
 
-                        local->success_count++;
-                }
+    local->inode = inode_ref(loc->inode);
 
-                local->op_errno = op_errno;
+    if (xattr_req) {
+        /* If xattr_req was null, afr_lookup_xattr_req_prepare() will
+           allocate one for us */
+        local->xattr_req = dict_copy_with_ref(xattr_req, NULL);
+        if (!local->xattr_req) {
+            op_errno = ENOMEM;
+            goto out;
+        }
+        ret = dict_get_gfuuid(local->xattr_req, "gfid-req",
+                              &local->cont.lookup.gfid_req);
+        if (ret == 0) {
+            dict_del_sizen(local->xattr_req, "gfid-req");
         }
-        UNLOCK (&frame->lock);
+    }
 
-        call_count = afr_frame_return (frame);
+    afr_read_subvol_get(loc->parent, this, NULL, NULL, &event,
+                        AFR_DATA_TRANSACTION, NULL);
 
-        if (call_count == 0) {
-                local->cont.fsync.prebuf.ia_ino  = local->cont.fsync.ino;
-                local->cont.fsync.postbuf.ia_ino = local->cont.fsync.ino;
+    afr_lookup_do(frame, this, 0);
 
-                AFR_STACK_UNWIND (fsync, frame, local->op_ret, local->op_errno,
-                                  &local->cont.fsync.prebuf,
-                                  &local->cont.fsync.postbuf);
-        }
+    return 0;
+out:
+    AFR_STACK_UNWIND(lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
+void
+_afr_cleanup_fd_ctx(xlator_t *this, afr_fd_ctx_t *fd_ctx)
+{
+    afr_private_t *priv = this->private;
+
+    if (fd_ctx->lk_heal_info) {
+        LOCK(&priv->lock);
+        {
+            list_del(&fd_ctx->lk_heal_info->pos);
+        }
+        afr_lk_heal_info_cleanup(fd_ctx->lk_heal_info);
+        fd_ctx->lk_heal_info = NULL;
+    }
+    GF_FREE(fd_ctx->opened_on);
+    GF_FREE(fd_ctx);
+    return;
+}
 
 int
-afr_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd,
-           int32_t datasync)
+afr_cleanup_fd_ctx(xlator_t *this, fd_t *fd)
 {
-        afr_private_t *priv = NULL;
-        afr_local_t *local = NULL;
+    uint64_t ctx = 0;
+    afr_fd_ctx_t *fd_ctx = NULL;
+    int ret = 0;
 
-        int ret = -1;
+    ret = fd_ctx_get(fd, this, &ctx);
+    if (ret < 0)
+        goto out;
 
-        int i = 0;
-        int32_t call_count = 0;
-        int32_t op_ret   = -1;
-        int32_t op_errno = 0;
+    fd_ctx = (afr_fd_ctx_t *)(long)ctx;
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+    if (fd_ctx) {
+        _afr_cleanup_fd_ctx(this, fd_ctx);
+    }
 
-        priv = this->private;
+out:
+    return 0;
+}
 
-        ALLOC_OR_GOTO (local, afr_local_t, out);
+int
+afr_release(xlator_t *this, fd_t *fd)
+{
+    afr_cleanup_fd_ctx(this, fd);
 
-        ret = AFR_LOCAL_INIT (local, priv);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
-        }
+    return 0;
+}
 
-        call_count = local->call_count;
-        frame->local = local;
+afr_fd_ctx_t *
+__afr_fd_ctx_get(fd_t *fd, xlator_t *this)
+{
+    uint64_t ctx = 0;
+    int ret = 0;
+    afr_fd_ctx_t *fd_ctx = NULL;
 
-        local->fd             = fd_ref (fd);
-        local->cont.fsync.ino = fd->inode->ino;
+    ret = __fd_ctx_get(fd, this, &ctx);
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, afr_fsync_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->fsync,
-                                           fd, datasync);
-                        if (!--call_count)
-                                break;
-                }
-        }
+    if (ret < 0) {
+        ret = __afr_fd_ctx_set(this, fd);
+        if (ret < 0)
+            goto out;
 
-        op_ret = 0;
+        ret = __fd_ctx_get(fd, this, &ctx);
+        if (ret < 0)
+            goto out;
+    }
+
+    fd_ctx = (afr_fd_ctx_t *)(long)ctx;
 out:
-        if (op_ret == -1) {
-                AFR_STACK_UNWIND (fsync, frame, op_ret, op_errno, NULL, NULL);
-        }
-        return 0;
+    return fd_ctx;
 }
 
-/* }}} */
+afr_fd_ctx_t *
+afr_fd_ctx_get(fd_t *fd, xlator_t *this)
+{
+    afr_fd_ctx_t *fd_ctx = NULL;
 
-/* {{{ fsync */
+    LOCK(&fd->lock);
+    {
+        fd_ctx = __afr_fd_ctx_get(fd, this);
+    }
+    UNLOCK(&fd->lock);
 
-int32_t
-afr_fsyncdir_cbk (call_frame_t *frame, void *cookie,
-                  xlator_t *this, int32_t op_ret, int32_t op_errno)
+    return fd_ctx;
+}
+
+int
+__afr_fd_ctx_set(xlator_t *this, fd_t *fd)
 {
-        afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int ret = -1;
+    uint64_t ctx = 0;
+    afr_fd_ctx_t *fd_ctx = NULL;
+    int i = 0;
+
+    VALIDATE_OR_GOTO(this->private, out);
+    VALIDATE_OR_GOTO(fd, out);
+
+    priv = this->private;
+
+    ret = __fd_ctx_get(fd, this, &ctx);
+
+    if (ret == 0)
+        goto out;
+
+    fd_ctx = GF_CALLOC(1, sizeof(afr_fd_ctx_t), gf_afr_mt_afr_fd_ctx_t);
+    if (!fd_ctx) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    fd_ctx->opened_on = GF_CALLOC(sizeof(*fd_ctx->opened_on), priv->child_count,
+                                  gf_afr_mt_int32_t);
+    if (!fd_ctx->opened_on) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (fd_is_anonymous(fd))
+            fd_ctx->opened_on[i] = AFR_FD_OPENED;
+        else
+            fd_ctx->opened_on[i] = AFR_FD_NOT_OPENED;
+    }
 
-        int call_count = -1;
+    fd_ctx->readdir_subvol = -1;
+    fd_ctx->lk_heal_info = NULL;
 
-        local = frame->local;
+    ret = __fd_ctx_set(fd, this, (uint64_t)(long)fd_ctx);
+    if (ret)
+        gf_msg_debug(this->name, 0, "failed to set fd ctx (%p)", fd);
+out:
+    if (ret && fd_ctx)
+        _afr_cleanup_fd_ctx(this, fd_ctx);
+    return ret;
+}
 
-        LOCK (&frame->lock);
-        {
-                if (op_ret == 0)
-                        local->op_ret = 0;
+/* {{{ flush */
+
+int
+afr_flush_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    int call_count = -1;
+
+    local = frame->local;
 
-                local->op_errno = op_errno;
+    LOCK(&frame->lock);
+    {
+        if (op_ret != -1) {
+            local->op_ret = op_ret;
+            if (!local->xdata_rsp && xdata)
+                local->xdata_rsp = dict_ref(xdata);
+        } else {
+            local->op_errno = op_errno;
         }
-        UNLOCK (&frame->lock);
+        call_count = --local->call_count;
+    }
+    UNLOCK(&frame->lock);
 
-        call_count = afr_frame_return (frame);
+    if (call_count == 0)
+        AFR_STACK_UNWIND(flush, frame, local->op_ret, local->op_errno,
+                         local->xdata_rsp);
 
-        if (call_count == 0)
-                AFR_STACK_UNWIND (fsyncdir, frame, local->op_ret,
-                                  local->op_errno);
+    return 0;
+}
 
-        return 0;
+static int
+afr_flush_wrapper(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+{
+    int i = 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int call_count = -1;
+
+    priv = this->private;
+    local = frame->local;
+    call_count = local->call_count;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->child_up[i]) {
+            STACK_WIND_COOKIE(frame, afr_flush_cbk, (void *)(long)i,
+                              priv->children[i], priv->children[i]->fops->flush,
+                              local->fd, xdata);
+            if (!--call_count)
+                break;
+        }
+    }
+
+    return 0;
 }
 
+afr_local_t *
+afr_wakeup_same_fd_delayed_op(xlator_t *this, afr_lock_t *lock, fd_t *fd)
+{
+    afr_local_t *local = NULL;
+
+    if (lock->delay_timer) {
+        local = list_entry(lock->post_op.next, afr_local_t,
+                           transaction.owner_list);
+        if (fd == local->fd) {
+            if (gf_timer_call_cancel(this->ctx, lock->delay_timer)) {
+                local = NULL;
+            } else {
+                lock->delay_timer = NULL;
+            }
+        } else {
+            local = NULL;
+        }
+    }
 
-int32_t
-afr_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
-              int32_t datasync)
+    return local;
+}
+
+void
+afr_delayed_changelog_wake_resume(xlator_t *this, inode_t *inode,
+                                  call_stub_t *stub)
+{
+    afr_inode_ctx_t *ctx = NULL;
+    afr_lock_t *lock = NULL;
+    afr_local_t *metadata_local = NULL;
+    afr_local_t *data_local = NULL;
+    LOCK(&inode->lock);
+    {
+        (void)__afr_inode_ctx_get(this, inode, &ctx);
+        lock = &ctx->lock[AFR_DATA_TRANSACTION];
+        data_local = afr_wakeup_same_fd_delayed_op(this, lock, stub->args.fd);
+        lock = &ctx->lock[AFR_METADATA_TRANSACTION];
+        metadata_local = afr_wakeup_same_fd_delayed_op(this, lock,
+                                                       stub->args.fd);
+    }
+    UNLOCK(&inode->lock);
+
+    if (data_local) {
+        data_local->transaction.resume_stub = stub;
+    } else if (metadata_local) {
+        metadata_local->transaction.resume_stub = stub;
+    } else {
+        call_resume(stub);
+    }
+    if (data_local) {
+        afr_delayed_changelog_wake_up_cbk(data_local);
+    }
+    if (metadata_local) {
+        afr_delayed_changelog_wake_up_cbk(metadata_local);
+    }
+}
+
+int
+afr_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        afr_private_t *priv = NULL;
-        afr_local_t *local = NULL;
+    afr_local_t *local = NULL;
+    call_stub_t *stub = NULL;
+    int op_errno = ENOMEM;
 
-        int ret = -1;
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-        int i = 0;
-        int32_t call_count = 0;
-        int32_t op_ret   = -1;
-        int32_t op_errno = 0;
+    local->op = GF_FOP_FLUSH;
+    if (!afr_is_consistent_io_possible(local, this->private, &op_errno))
+        goto out;
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+    local->fd = fd_ref(fd);
 
-        priv = this->private;
+    stub = fop_flush_stub(frame, afr_flush_wrapper, fd, xdata);
+    if (!stub)
+        goto out;
 
-        ALLOC_OR_GOTO (local, afr_local_t, out);
+    afr_delayed_changelog_wake_resume(this, fd->inode, stub);
 
-        ret = AFR_LOCAL_INIT (local, priv);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
+    return 0;
+out:
+    AFR_STACK_UNWIND(flush, frame, -1, op_errno, NULL);
+    return 0;
+}
+
+int
+afr_fsyncdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    int call_count = -1;
+
+    local = frame->local;
+
+    LOCK(&frame->lock);
+    {
+        if (op_ret == 0) {
+            local->op_ret = 0;
+            if (!local->xdata_rsp && xdata)
+                local->xdata_rsp = dict_ref(xdata);
+        } else {
+            local->op_errno = op_errno;
         }
+        call_count = --local->call_count;
+    }
+    UNLOCK(&frame->lock);
 
-        call_count = local->call_count;
-        frame->local = local;
+    if (call_count == 0)
+        AFR_STACK_UNWIND(fsyncdir, frame, local->op_ret, local->op_errno,
+                         local->xdata_rsp);
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND (frame, afr_fsyncdir_cbk,
-                                    priv->children[i],
-                                    priv->children[i]->fops->fsyncdir,
-                                    fd, datasync);
-                        if (!--call_count)
-                                break;
-                }
+    return 0;
+}
+
+int
+afr_fsyncdir(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+             dict_t *xdata)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int i = 0;
+    int32_t call_count = 0;
+    int32_t op_errno = ENOMEM;
+
+    priv = this->private;
+
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
+
+    local->op = GF_FOP_FSYNCDIR;
+    if (!afr_is_consistent_io_possible(local, priv, &op_errno))
+        goto out;
+
+    call_count = local->call_count;
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->child_up[i]) {
+            STACK_WIND(frame, afr_fsyncdir_cbk, priv->children[i],
+                       priv->children[i]->fops->fsyncdir, fd, datasync, xdata);
+            if (!--call_count)
+                break;
         }
+    }
 
-        op_ret = 0;
+    return 0;
 out:
-        if (op_ret == -1) {
-                AFR_STACK_UNWIND (fsyncdir, frame, op_ret, op_errno);
-        }
-        return 0;
+    AFR_STACK_UNWIND(fsyncdir, frame, -1, op_errno, NULL);
+
+    return 0;
 }
 
 /* }}} */
 
-/* {{{ xattrop */
+static int
+afr_serialized_lock_wind(call_frame_t *frame, xlator_t *this);
 
-int32_t
-afr_xattrop_cbk (call_frame_t *frame, void *cookie,
-                 xlator_t *this, int32_t op_ret, int32_t op_errno,
-                 dict_t *xattr)
+static gf_boolean_t
+afr_is_conflicting_lock_present(int32_t op_ret, int32_t op_errno)
 {
-        afr_local_t *local = NULL;
+    if (op_ret == -1 && op_errno == EAGAIN)
+        return _gf_true;
+    return _gf_false;
+}
+
+static void
+afr_fop_lock_unwind(call_frame_t *frame, glusterfs_fop_t op, int32_t op_ret,
+                    int32_t op_errno, dict_t *xdata)
+{
+    switch (op) {
+        case GF_FOP_INODELK:
+            AFR_STACK_UNWIND(inodelk, frame, op_ret, op_errno, xdata);
+            break;
+        case GF_FOP_FINODELK:
+            AFR_STACK_UNWIND(finodelk, frame, op_ret, op_errno, xdata);
+            break;
+        case GF_FOP_ENTRYLK:
+            AFR_STACK_UNWIND(entrylk, frame, op_ret, op_errno, xdata);
+            break;
+        case GF_FOP_FENTRYLK:
+            AFR_STACK_UNWIND(fentrylk, frame, op_ret, op_errno, xdata);
+            break;
+        default:
+            break;
+    }
+}
 
-        int call_count = -1;
+static void
+afr_fop_lock_wind(call_frame_t *frame, xlator_t *this, int child_index,
+                  int32_t (*lock_cbk)(call_frame_t *, void *, xlator_t *,
+                                      int32_t, int32_t, dict_t *))
+{
+    afr_local_t *local = frame->local;
+    afr_private_t *priv = this->private;
+    int i = child_index;
+
+    switch (local->op) {
+        case GF_FOP_INODELK:
+            STACK_WIND_COOKIE(
+                frame, lock_cbk, (void *)(long)i, priv->children[i],
+                priv->children[i]->fops->inodelk,
+                (const char *)local->cont.inodelk.volume, &local->loc,
+                local->cont.inodelk.cmd, &local->cont.inodelk.flock,
+                local->cont.inodelk.xdata);
+            break;
+        case GF_FOP_FINODELK:
+            STACK_WIND_COOKIE(
+                frame, lock_cbk, (void *)(long)i, priv->children[i],
+                priv->children[i]->fops->finodelk,
+                (const char *)local->cont.inodelk.volume, local->fd,
+                local->cont.inodelk.cmd, &local->cont.inodelk.flock,
+                local->cont.inodelk.xdata);
+            break;
+        case GF_FOP_ENTRYLK:
+            STACK_WIND_COOKIE(
+                frame, lock_cbk, (void *)(long)i, priv->children[i],
+                priv->children[i]->fops->entrylk, local->cont.entrylk.volume,
+                &local->loc, local->cont.entrylk.basename,
+                local->cont.entrylk.cmd, local->cont.entrylk.type,
+                local->cont.entrylk.xdata);
+            break;
+        case GF_FOP_FENTRYLK:
+            STACK_WIND_COOKIE(
+                frame, lock_cbk, (void *)(long)i, priv->children[i],
+                priv->children[i]->fops->fentrylk, local->cont.entrylk.volume,
+                local->fd, local->cont.entrylk.basename,
+                local->cont.entrylk.cmd, local->cont.entrylk.type,
+                local->cont.entrylk.xdata);
+            break;
+        default:
+            break;
+    }
+}
 
-        local = frame->local;
+void
+afr_fop_lock_proceed(call_frame_t *frame)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        LOCK (&frame->lock);
-        {
-                if (op_ret == 0)
-                        local->op_ret = 0;
+    local = frame->local;
+    priv = frame->this->private;
 
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
+    if (local->fop_lock_state != AFR_FOP_LOCK_PARALLEL) {
+        afr_fop_lock_unwind(frame, local->op, local->op_ret, local->op_errno,
+                            local->xdata_rsp);
+        return;
+    }
+    /* At least one child is up */
+    /*
+     * Non-blocking locks also need to be serialized.  Otherwise there is
+     * a chance that both the mounts which issued same non-blocking inodelk
+     * may endup not acquiring the lock on any-brick.
+     * Ex: Mount1 and Mount2
+     * request for full length lock on file f1.  Mount1 afr may acquire the
+     * partial lock on brick-1 and may not acquire the lock on brick-2
+     * because Mount2 already got the lock on brick-2, vice versa.  Since
+     * both the mounts only got partial locks, afr treats them as failure in
+     * gaining the locks and unwinds with EAGAIN errno.
+     */
+    local->op_ret = -1;
+    local->op_errno = EUCLEAN;
+    local->fop_lock_state = AFR_FOP_LOCK_SERIAL;
+    afr_local_replies_wipe(local, priv);
+    if (local->xdata_rsp)
+        dict_unref(local->xdata_rsp);
+    local->xdata_rsp = NULL;
+    switch (local->op) {
+        case GF_FOP_INODELK:
+        case GF_FOP_FINODELK:
+            local->cont.inodelk.cmd = local->cont.inodelk.in_cmd;
+            local->cont.inodelk.flock = local->cont.inodelk.in_flock;
+            if (local->cont.inodelk.xdata)
+                dict_unref(local->cont.inodelk.xdata);
+            local->cont.inodelk.xdata = NULL;
+            if (local->xdata_req)
+                local->cont.inodelk.xdata = dict_ref(local->xdata_req);
+            break;
+        case GF_FOP_ENTRYLK:
+        case GF_FOP_FENTRYLK:
+            local->cont.entrylk.cmd = local->cont.entrylk.in_cmd;
+            if (local->cont.entrylk.xdata)
+                dict_unref(local->cont.entrylk.xdata);
+            local->cont.entrylk.xdata = NULL;
+            if (local->xdata_req)
+                local->cont.entrylk.xdata = dict_ref(local->xdata_req);
+            break;
+        default:
+            break;
+    }
+    afr_serialized_lock_wind(frame, frame->this);
+}
 
-        call_count = afr_frame_return (frame);
+static int32_t
+afr_unlock_partial_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
-        if (call_count == 0)
-                AFR_STACK_UNWIND (xattrop, frame, local->op_ret, local->op_errno,
-                                  xattr);
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int call_count = -1;
+    int child_index = (long)cookie;
+    uuid_t gfid = {0};
 
-        return 0;
+    local = frame->local;
+    priv = this->private;
+
+    if (op_ret < 0 && op_errno != ENOTCONN) {
+        if (local->fd)
+            gf_uuid_copy(gfid, local->fd->inode->gfid);
+        else
+            loc_gfid(&local->loc, gfid);
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, AFR_MSG_UNLOCK_FAIL,
+               "%s: Failed to unlock %s on %s "
+               "with lk_owner: %s",
+               uuid_utoa(gfid), gf_fop_list[local->op],
+               priv->children[child_index]->name,
+               lkowner_utoa(&frame->root->lk_owner));
+    }
+
+    call_count = afr_frame_return(frame);
+    if (call_count == 0)
+        afr_fop_lock_proceed(frame);
+
+    return 0;
 }
 
+static int32_t
+afr_unlock_locks_and_proceed(call_frame_t *frame, xlator_t *this,
+                             int call_count)
+{
+    int i = 0;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+
+    if (call_count == 0) {
+        afr_fop_lock_proceed(frame);
+        goto out;
+    }
+
+    local = frame->local;
+    priv = this->private;
+    local->call_count = call_count;
+    switch (local->op) {
+        case GF_FOP_INODELK:
+        case GF_FOP_FINODELK:
+            local->cont.inodelk.flock.l_type = F_UNLCK;
+            local->cont.inodelk.cmd = F_SETLK;
+            if (local->cont.inodelk.xdata)
+                dict_unref(local->cont.inodelk.xdata);
+            local->cont.inodelk.xdata = NULL;
+            break;
+        case GF_FOP_ENTRYLK:
+        case GF_FOP_FENTRYLK:
+            local->cont.entrylk.cmd = ENTRYLK_UNLOCK;
+            if (local->cont.entrylk.xdata)
+                dict_unref(local->cont.entrylk.xdata);
+            local->cont.entrylk.xdata = NULL;
+            break;
+        default:
+            break;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid)
+            continue;
+
+        if (local->replies[i].op_ret == -1)
+            continue;
+
+        afr_fop_lock_wind(frame, this, i, afr_unlock_partial_lock_cbk);
+
+        if (!--call_count)
+            break;
+    }
+
+out:
+    return 0;
+}
 
 int32_t
-afr_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             gf_xattrop_flags_t optype, dict_t *xattr)
+afr_fop_lock_done(call_frame_t *frame, xlator_t *this)
 {
-        afr_private_t *priv = NULL;
-        afr_local_t *local  = NULL;
+    int i = 0;
+    int lock_count = 0;
+    unsigned char *success = NULL;
 
-        int ret = -1;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        int i = 0;
-        int32_t call_count = 0;
-        int32_t op_ret   = -1;
-        int32_t op_errno = 0;
+    local = frame->local;
+    priv = this->private;
+    success = alloca0(priv->child_count);
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid)
+            continue;
 
-        priv = this->private;
+        if (local->replies[i].op_ret == 0) {
+            lock_count++;
+            success[i] = 1;
+        }
 
-        ALLOC_OR_GOTO (local, afr_local_t, out);
+        if (local->op_ret == -1 && local->op_errno == EAGAIN)
+            continue;
 
-        ret = AFR_LOCAL_INIT (local, priv);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
+        if ((local->replies[i].op_ret == -1) &&
+            (local->replies[i].op_errno == EAGAIN)) {
+            local->op_ret = -1;
+            local->op_errno = EAGAIN;
+            continue;
         }
 
-        call_count = local->call_count;
-        frame->local = local;
+        if (local->replies[i].op_ret == 0)
+            local->op_ret = 0;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND (frame, afr_xattrop_cbk,
-                                    priv->children[i],
-                                    priv->children[i]->fops->xattrop,
-                                    loc, optype, xattr);
-                        if (!--call_count)
-                                break;
-                }
+        local->op_errno = local->replies[i].op_errno;
+    }
+
+    if (afr_fop_lock_is_unlock(frame))
+        goto unwind;
+
+    if (afr_is_conflicting_lock_present(local->op_ret, local->op_errno)) {
+        afr_unlock_locks_and_proceed(frame, this, lock_count);
+    } else if (priv->quorum_count && !afr_has_quorum(success, this, NULL)) {
+        local->fop_lock_state = AFR_FOP_LOCK_QUORUM_FAILED;
+        local->op_ret = -1;
+        local->op_errno = afr_final_errno(local, priv);
+        if (local->op_errno == 0)
+            local->op_errno = afr_quorum_errno(priv);
+        afr_unlock_locks_and_proceed(frame, this, lock_count);
+    } else {
+        goto unwind;
+    }
+
+    return 0;
+unwind:
+    afr_fop_lock_unwind(frame, local->op, local->op_ret, local->op_errno,
+                        local->xdata_rsp);
+    return 0;
+}
+
+static int
+afr_common_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    int child_index = (long)cookie;
+
+    local = frame->local;
+
+    local->replies[child_index].valid = 1;
+    local->replies[child_index].op_ret = op_ret;
+    local->replies[child_index].op_errno = op_errno;
+    if (op_ret == 0 && xdata) {
+        local->replies[child_index].xdata = dict_ref(xdata);
+        LOCK(&frame->lock);
+        {
+            if (!local->xdata_rsp)
+                local->xdata_rsp = dict_ref(xdata);
         }
+        UNLOCK(&frame->lock);
+    }
+    return 0;
+}
 
-        op_ret = 0;
-out:
-        if (op_ret == -1) {
-                AFR_STACK_UNWIND (xattrop, frame, op_ret, op_errno, NULL);
+static int32_t
+afr_serialized_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int child_index = (long)cookie;
+    int next_child = 0;
+
+    local = frame->local;
+    priv = this->private;
+
+    afr_common_lock_cbk(frame, cookie, this, op_ret, op_errno, xdata);
+
+    for (next_child = child_index + 1; next_child < priv->child_count;
+         next_child++) {
+        if (local->child_up[next_child])
+            break;
+    }
+
+    if (afr_is_conflicting_lock_present(op_ret, op_errno) ||
+        (next_child == priv->child_count)) {
+        afr_fop_lock_done(frame, this);
+    } else {
+        afr_fop_lock_wind(frame, this, next_child, afr_serialized_lock_cbk);
+    }
+
+    return 0;
+}
+
+static int
+afr_serialized_lock_wind(call_frame_t *frame, xlator_t *this)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int i = 0;
+
+    priv = this->private;
+    local = frame->local;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->child_up[i]) {
+            afr_fop_lock_wind(frame, this, i, afr_serialized_lock_cbk);
+            break;
         }
-        return 0;
+    }
+    return 0;
 }
 
-/* }}} */
+static int32_t
+afr_parallel_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
-/* {{{ fxattrop */
+{
+    int call_count = 0;
 
-int32_t
-afr_fxattrop_cbk (call_frame_t *frame, void *cookie,
-                  xlator_t *this, int32_t op_ret, int32_t op_errno,
-                  dict_t *xattr)
+    afr_common_lock_cbk(frame, cookie, this, op_ret, op_errno, xdata);
+
+    call_count = afr_frame_return(frame);
+    if (call_count == 0)
+        afr_fop_lock_done(frame, this);
+
+    return 0;
+}
+
+static int
+afr_parallel_lock_wind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int call_count = 0;
+    int i = 0;
+
+    priv = this->private;
+    local = frame->local;
+    call_count = local->call_count;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->child_up[i])
+            continue;
+        afr_fop_lock_wind(frame, this, i, afr_parallel_lock_cbk);
+        if (!--call_count)
+            break;
+    }
+    return 0;
+}
 
-        int call_count = -1;
+static int
+afr_fop_handle_lock(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = frame->local;
+    int op_errno = 0;
 
-        local = frame->local;
+    if (!afr_fop_lock_is_unlock(frame)) {
+        if (!afr_is_consistent_io_possible(local, this->private, &op_errno))
+            goto out;
 
-        LOCK (&frame->lock);
-        {
-                if (op_ret == 0)
-                        local->op_ret = 0;
+        switch (local->op) {
+            case GF_FOP_INODELK:
+            case GF_FOP_FINODELK:
+                local->cont.inodelk.cmd = F_SETLK;
+                break;
+            case GF_FOP_ENTRYLK:
+            case GF_FOP_FENTRYLK:
+                local->cont.entrylk.cmd = ENTRYLK_LOCK_NB;
+                break;
+            default:
+                break;
+        }
+    }
 
-                local->op_errno = op_errno;
+    if (local->xdata_req) {
+        switch (local->op) {
+            case GF_FOP_INODELK:
+            case GF_FOP_FINODELK:
+                local->cont.inodelk.xdata = dict_ref(local->xdata_req);
+                break;
+            case GF_FOP_ENTRYLK:
+            case GF_FOP_FENTRYLK:
+                local->cont.entrylk.xdata = dict_ref(local->xdata_req);
+                break;
+            default:
+                break;
         }
-        UNLOCK (&frame->lock);
+    }
 
-        call_count = afr_frame_return (frame);
+    local->fop_lock_state = AFR_FOP_LOCK_PARALLEL;
+    afr_parallel_lock_wind(frame, this);
+out:
+    return -op_errno;
+}
 
-        if (call_count == 0)
-                AFR_STACK_UNWIND (fxattrop, frame, local->op_ret, local->op_errno,
-                                  xattr);
+static int32_t
+afr_handle_inodelk(call_frame_t *frame, xlator_t *this, glusterfs_fop_t fop,
+                   const char *volume, loc_t *loc, fd_t *fd, int32_t cmd,
+                   struct gf_flock *flock, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    int32_t op_errno = ENOMEM;
+
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
+
+    local->op = fop;
+    if (loc)
+        loc_copy(&local->loc, loc);
+    if (fd && (flock->l_type != F_UNLCK)) {
+        AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+        local->fd = fd_ref(fd);
+    }
+
+    local->cont.inodelk.volume = gf_strdup(volume);
+    if (!local->cont.inodelk.volume) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    local->cont.inodelk.in_cmd = cmd;
+    local->cont.inodelk.cmd = cmd;
+    local->cont.inodelk.in_flock = *flock;
+    local->cont.inodelk.flock = *flock;
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
+
+    op_errno = -afr_fop_handle_lock(frame, frame->this);
+    if (op_errno)
+        goto out;
+    return 0;
+out:
+    afr_fop_lock_unwind(frame, fop, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
+int32_t
+afr_inodelk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
+            int32_t cmd, struct gf_flock *flock, dict_t *xdata)
+{
+    afr_handle_inodelk(frame, this, GF_FOP_INODELK, volume, loc, NULL, cmd,
+                       flock, xdata);
+    return 0;
+}
 
 int32_t
-afr_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
-              gf_xattrop_flags_t optype, dict_t *xattr)
+afr_finodelk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+             int32_t cmd, struct gf_flock *flock, dict_t *xdata)
 {
-        afr_private_t *priv = NULL;
-        afr_local_t *local  = NULL;
+    afr_handle_inodelk(frame, this, GF_FOP_FINODELK, volume, NULL, fd, cmd,
+                       flock, xdata);
+    return 0;
+}
 
-        int ret = -1;
+static int
+afr_handle_entrylk(call_frame_t *frame, xlator_t *this, glusterfs_fop_t fop,
+                   const char *volume, loc_t *loc, fd_t *fd,
+                   const char *basename, entrylk_cmd cmd, entrylk_type type,
+                   dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    int32_t op_errno = ENOMEM;
+
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
+
+    local->op = fop;
+    if (loc)
+        loc_copy(&local->loc, loc);
+    if (fd && (cmd != ENTRYLK_UNLOCK)) {
+        AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+        local->fd = fd_ref(fd);
+    }
+    local->cont.entrylk.cmd = cmd;
+    local->cont.entrylk.in_cmd = cmd;
+    local->cont.entrylk.type = type;
+    local->cont.entrylk.volume = gf_strdup(volume);
+    local->cont.entrylk.basename = gf_strdup(basename);
+    if (!local->cont.entrylk.volume || !local->cont.entrylk.basename) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
+    op_errno = -afr_fop_handle_lock(frame, frame->this);
+    if (op_errno)
+        goto out;
+
+    return 0;
+out:
+    afr_fop_lock_unwind(frame, fop, -1, op_errno, NULL);
+    return 0;
+}
 
-        int i = 0;
-        int32_t call_count = 0;
-        int32_t op_ret   = -1;
-        int32_t op_errno = 0;
+int
+afr_entrylk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
+            const char *basename, entrylk_cmd cmd, entrylk_type type,
+            dict_t *xdata)
+{
+    afr_handle_entrylk(frame, this, GF_FOP_ENTRYLK, volume, loc, NULL, basename,
+                       cmd, type, xdata);
+    return 0;
+}
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+int
+afr_fentrylk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+             const char *basename, entrylk_cmd cmd, entrylk_type type,
+             dict_t *xdata)
+{
+    afr_handle_entrylk(frame, this, GF_FOP_FENTRYLK, volume, NULL, fd, basename,
+                       cmd, type, xdata);
+    return 0;
+}
 
-        priv = this->private;
+int
+afr_statfs_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, struct statvfs *statvfs, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    int call_count = 0;
+    struct statvfs *buf = NULL;
 
-        ALLOC_OR_GOTO (local, afr_local_t, out);
+    local = frame->local;
 
-        ret = AFR_LOCAL_INIT (local, priv);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
+    LOCK(&frame->lock);
+    {
+        if (op_ret != 0) {
+            local->op_errno = op_errno;
+            goto unlock;
         }
 
-        call_count = local->call_count;
-        frame->local = local;
+        local->op_ret = op_ret;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND (frame, afr_fxattrop_cbk,
-                                    priv->children[i],
-                                    priv->children[i]->fops->fxattrop,
-                                    fd, optype, xattr);
-                        if (!--call_count)
-                                break;
+        buf = &local->cont.statfs.buf;
+        if (local->cont.statfs.buf_set) {
+            if (statvfs->f_bavail < buf->f_bavail) {
+                *buf = *statvfs;
+                if (xdata) {
+                    if (local->xdata_rsp)
+                        dict_unref(local->xdata_rsp);
+                    local->xdata_rsp = dict_ref(xdata);
                 }
+            }
+        } else {
+            *buf = *statvfs;
+            local->cont.statfs.buf_set = 1;
+            if (xdata)
+                local->xdata_rsp = dict_ref(xdata);
         }
+    }
+unlock:
+    call_count = --local->call_count;
+    UNLOCK(&frame->lock);
 
-        op_ret = 0;
-out:
-        if (op_ret == -1) {
-                AFR_STACK_UNWIND (fxattrop, frame, op_ret, op_errno, NULL);
-        }
-        return 0;
+    if (call_count == 0)
+        AFR_STACK_UNWIND(statfs, frame, local->op_ret, local->op_errno,
+                         &local->cont.statfs.buf, local->xdata_rsp);
+
+    return 0;
 }
 
-/* }}} */
+int
+afr_statfs(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    int call_count = 0;
+    int32_t op_errno = ENOMEM;
+
+    priv = this->private;
+
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
+
+    local->op = GF_FOP_STATFS;
+    if (!afr_is_consistent_io_possible(local, priv, &op_errno))
+        goto out;
+
+    if (priv->arbiter_count == 1 && local->child_up[ARBITER_BRICK_INDEX])
+        local->call_count--;
+    call_count = local->call_count;
+    if (!call_count) {
+        op_errno = ENOTCONN;
+        goto out;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->child_up[i]) {
+            if (AFR_IS_ARBITER_BRICK(priv, i))
+                continue;
+            STACK_WIND(frame, afr_statfs_cbk, priv->children[i],
+                       priv->children[i]->fops->statfs, loc, xdata);
+            if (!--call_count)
+                break;
+        }
+    }
 
+    return 0;
+out:
+    AFR_STACK_UNWIND(statfs, frame, -1, op_errno, NULL, NULL);
+
+    return 0;
+}
 
 int32_t
-afr_inodelk_cbk (call_frame_t *frame, void *cookie,
-                 xlator_t *this, int32_t op_ret, int32_t op_errno)
+afr_lk_unlock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+                  dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = this->private;
+    int call_count = -1;
+    int child_index = (long)cookie;
+
+    local = frame->local;
+
+    if (op_ret < 0 && op_errno != ENOTCONN && op_errno != EBADFD) {
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, AFR_MSG_UNLOCK_FAIL,
+               "gfid=%s: unlock failed on subvolume %s "
+               "with lock owner %s",
+               uuid_utoa(local->fd->inode->gfid),
+               priv->children[child_index]->name,
+               lkowner_utoa(&frame->root->lk_owner));
+    }
+
+    call_count = afr_frame_return(frame);
+    if (call_count == 0) {
+        AFR_STACK_UNWIND(lk, frame, local->op_ret, local->op_errno, NULL,
+                         local->xdata_rsp);
+    }
+
+    return 0;
+}
 
+int32_t
+afr_lk_unlock(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *local = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    int call_count = 0;
 
-        int call_count = -1;
+    local = frame->local;
+    priv = this->private;
 
-        local = frame->local;
+    call_count = afr_locked_nodes_count(local->cont.lk.locked_nodes,
+                                        priv->child_count);
 
-        LOCK (&frame->lock);
-        {
-                if (op_ret == 0)
-                        local->op_ret = 0;
+    if (call_count == 0) {
+        AFR_STACK_UNWIND(lk, frame, local->op_ret, local->op_errno, NULL,
+                         local->xdata_rsp);
+        return 0;
+    }
 
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
+    local->call_count = call_count;
 
-        call_count = afr_frame_return (frame);
+    local->cont.lk.user_flock.l_type = F_UNLCK;
 
-        if (call_count == 0)
-                AFR_STACK_UNWIND (inodelk, frame, local->op_ret,
-                                  local->op_errno);
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->cont.lk.locked_nodes[i]) {
+            STACK_WIND_COOKIE(frame, afr_lk_unlock_cbk, (void *)(long)i,
+                              priv->children[i], priv->children[i]->fops->lk,
+                              local->fd, F_SETLK, &local->cont.lk.user_flock,
+                              NULL);
 
-        return 0;
-}
+            if (!--call_count)
+                break;
+        }
+    }
 
+    return 0;
+}
 
 int32_t
-afr_inodelk (call_frame_t *frame, xlator_t *this,
-             const char *volume, loc_t *loc, int32_t cmd, struct gf_flock *flock)
+afr_lk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+           int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
 {
-        afr_private_t *priv = NULL;
-        afr_local_t *local  = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int child_index = -1;
 
-        int ret = -1;
+    local = frame->local;
+    priv = this->private;
 
-        int i = 0;
-        int32_t call_count = 0;
-        int32_t op_ret   = -1;
-        int32_t op_errno = 0;
+    child_index = (long)cookie;
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+    afr_common_lock_cbk(frame, cookie, this, op_ret, op_errno, xdata);
+    if (op_ret < 0 && op_errno == EAGAIN) {
+        local->op_ret = -1;
+        local->op_errno = EAGAIN;
 
-        priv = this->private;
+        afr_lk_unlock(frame, this);
+        return 0;
+    }
+
+    if (op_ret == 0) {
+        local->op_ret = 0;
+        local->op_errno = 0;
+        local->cont.lk.locked_nodes[child_index] = 1;
+        local->cont.lk.ret_flock = *lock;
+    }
+
+    child_index++;
+
+    if (child_index < priv->child_count) {
+        STACK_WIND_COOKIE(frame, afr_lk_cbk, (void *)(long)child_index,
+                          priv->children[child_index],
+                          priv->children[child_index]->fops->lk, local->fd,
+                          local->cont.lk.cmd, &local->cont.lk.user_flock,
+                          local->xdata_req);
+    } else if (priv->quorum_count &&
+               !afr_has_quorum(local->cont.lk.locked_nodes, this, NULL)) {
+        local->op_ret = -1;
+        local->op_errno = afr_final_errno(local, priv);
 
-        ALLOC_OR_GOTO (local, afr_local_t, out);
+        afr_lk_unlock(frame, this);
+    } else {
+        if (local->op_ret < 0)
+            local->op_errno = afr_final_errno(local, priv);
 
-        ret = AFR_LOCAL_INIT (local, priv);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
-        }
+        AFR_STACK_UNWIND(lk, frame, local->op_ret, local->op_errno,
+                         &local->cont.lk.ret_flock, local->xdata_rsp);
+    }
 
-        call_count = local->call_count;
-        frame->local = local;
+    return 0;
+}
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND (frame, afr_inodelk_cbk,
-                                    priv->children[i],
-                                    priv->children[i]->fops->inodelk,
-                                    volume, loc, cmd, flock);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
+int
+afr_lk_transaction_cbk(int ret, call_frame_t *frame, void *opaque)
+{
+    return 0;
+}
 
-        op_ret = 0;
-out:
-        if (op_ret == -1) {
-                AFR_STACK_UNWIND (inodelk, frame, op_ret, op_errno);
-        }
-        return 0;
+int
+afr_lk_txn_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+                    dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    int child_index = -1;
+
+    local = frame->local;
+    child_index = (long)cookie;
+    afr_common_lock_cbk(frame, cookie, this, op_ret, op_errno, xdata);
+    if (op_ret == 0) {
+        local->op_ret = 0;
+        local->op_errno = 0;
+        local->cont.lk.locked_nodes[child_index] = 1;
+        local->cont.lk.ret_flock = *lock;
+    }
+    syncbarrier_wake(&local->barrier);
+    return 0;
 }
 
+int
+afr_lk_txn_unlock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+                      dict_t *xdata)
+{
+    afr_local_t *local = frame->local;
+    afr_private_t *priv = this->private;
+    int child_index = (long)cookie;
+
+    if (op_ret < 0 && op_errno != ENOTCONN && op_errno != EBADFD) {
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, AFR_MSG_UNLOCK_FAIL,
+               "gfid=%s: unlock failed on subvolume %s "
+               "with lock owner %s",
+               uuid_utoa(local->fd->inode->gfid),
+               priv->children[child_index]->name,
+               lkowner_utoa(&frame->root->lk_owner));
+    }
+    return 0;
+}
+int
+afr_lk_transaction(void *opaque)
+{
+    call_frame_t *frame = NULL;
+    xlator_t *this = NULL;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    char *wind_on = NULL;
+    int op_errno = 0;
+    int i = 0;
+    int ret = 0;
+
+    frame = (call_frame_t *)opaque;
+    local = frame->local;
+    this = frame->this;
+    priv = this->private;
+    wind_on = alloca0(priv->child_count);
+
+    if (priv->arbiter_count || priv->child_count != 3) {
+        op_errno = ENOTSUP;
+        gf_msg(frame->this->name, GF_LOG_ERROR, op_errno, AFR_MSG_LK_HEAL_DOM,
+               "%s: Lock healing supported only for replica 3 volumes.",
+               uuid_utoa(local->fd->inode->gfid));
+        goto err;
+    }
+
+    op_errno = -afr_dom_lock_acquire(frame);  // Released during
+                                              // AFR_STACK_UNWIND
+    if (op_errno != 0) {
+        goto err;
+    }
+    if (priv->quorum_count &&
+        !afr_has_quorum(local->cont.lk.dom_locked_nodes, this, NULL)) {
+        op_errno = afr_final_errno(local, priv);
+        goto err;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (priv->child_up[i] && local->cont.lk.dom_locked_nodes[i])
+            wind_on[i] = 1;
+    }
+    AFR_ONLIST(wind_on, frame, afr_lk_txn_wind_cbk, lk, local->fd,
+               local->cont.lk.cmd, &local->cont.lk.user_flock,
+               local->xdata_req);
+
+    if (priv->quorum_count &&
+        !afr_has_quorum(local->cont.lk.locked_nodes, this, NULL)) {
+        local->op_ret = -1;
+        local->op_errno = afr_final_errno(local, priv);
+        goto unlock;
+    } else {
+        if (local->cont.lk.user_flock.l_type == F_UNLCK)
+            ret = afr_remove_lock_from_saved_locks(local, this);
+        else
+            ret = afr_add_lock_to_saved_locks(frame, this);
+        if (ret) {
+            local->op_ret = -1;
+            local->op_errno = -ret;
+            goto unlock;
+        }
+        AFR_STACK_UNWIND(lk, frame, local->op_ret, local->op_errno,
+                         &local->cont.lk.ret_flock, local->xdata_rsp);
+    }
 
-int32_t
-afr_finodelk_cbk (call_frame_t *frame, void *cookie,
-                  xlator_t *this, int32_t op_ret, int32_t op_errno)
+    return 0;
+
+unlock:
+    local->cont.lk.user_flock.l_type = F_UNLCK;
+    AFR_ONLIST(local->cont.lk.locked_nodes, frame, afr_lk_txn_unlock_cbk, lk,
+               local->fd, F_SETLK, &local->cont.lk.user_flock, NULL);
+err:
+    AFR_STACK_UNWIND(lk, frame, -1, op_errno, NULL, NULL);
+    return -1;
+}
 
+int
+afr_lk(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
+       struct gf_flock *flock, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int ret = 0;
+    int i = 0;
+    int32_t op_errno = ENOMEM;
+
+    priv = this->private;
+
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
+
+    local->op = GF_FOP_LK;
+    if (!afr_lk_is_unlock(cmd, flock)) {
+        AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+        if (!afr_is_consistent_io_possible(local, priv, &op_errno))
+            goto out;
+    }
+
+    local->cont.lk.locked_nodes = GF_CALLOC(
+        priv->child_count, sizeof(*local->cont.lk.locked_nodes),
+        gf_afr_mt_char);
+
+    if (!local->cont.lk.locked_nodes) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    local->fd = fd_ref(fd);
+    local->cont.lk.cmd = cmd;
+    local->cont.lk.user_flock = *flock;
+    local->cont.lk.ret_flock = *flock;
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
+
+    if (afr_is_lock_mode_mandatory(xdata)) {
+        ret = synctask_new(this->ctx->env, afr_lk_transaction,
+                           afr_lk_transaction_cbk, frame, frame);
+        if (ret) {
+            op_errno = ENOMEM;
+            goto out;
+        }
+        return 0;
+    }
 
-        int call_count = -1;
+    STACK_WIND_COOKIE(frame, afr_lk_cbk, (void *)(long)0, priv->children[i],
+                      priv->children[i]->fops->lk, fd, cmd, flock,
+                      local->xdata_req);
 
-        local = frame->local;
+    return 0;
+out:
+    AFR_STACK_UNWIND(lk, frame, -1, op_errno, NULL, NULL);
 
-        LOCK (&frame->lock);
-        {
-                if (op_ret == 0)
-                        local->op_ret = 0;
+    return 0;
+}
 
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
+int32_t
+afr_lease_unlock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct gf_lease *lease,
+                     dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    int call_count = -1;
 
-        call_count = afr_frame_return (frame);
+    local = frame->local;
+    call_count = afr_frame_return(frame);
 
-        if (call_count == 0)
-                AFR_STACK_UNWIND (finodelk, frame, local->op_ret,
-                                  local->op_errno);
+    if (call_count == 0)
+        AFR_STACK_UNWIND(lease, frame, local->op_ret, local->op_errno, lease,
+                         xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-afr_finodelk (call_frame_t *frame, xlator_t *this,
-              const char *volume, fd_t *fd, int32_t cmd, struct gf_flock *flock)
+afr_lease_unlock(call_frame_t *frame, xlator_t *this)
 {
-        afr_private_t *priv = NULL;
-        afr_local_t *local  = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    int call_count = 0;
 
-        int ret = -1;
+    local = frame->local;
+    priv = this->private;
 
-        int i = 0;
-        int32_t call_count = 0;
-        int32_t op_ret   = -1;
-        int32_t op_errno = 0;
+    call_count = afr_locked_nodes_count(local->cont.lease.locked_nodes,
+                                        priv->child_count);
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+    if (call_count == 0) {
+        AFR_STACK_UNWIND(lease, frame, local->op_ret, local->op_errno,
+                         &local->cont.lease.ret_lease, NULL);
+        return 0;
+    }
 
-        priv = this->private;
+    local->call_count = call_count;
 
-        ALLOC_OR_GOTO (local, afr_local_t, out);
+    local->cont.lease.user_lease.cmd = GF_UNLK_LEASE;
 
-        ret = AFR_LOCAL_INIT (local, priv);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->cont.lease.locked_nodes[i]) {
+            STACK_WIND(frame, afr_lease_unlock_cbk, priv->children[i],
+                       priv->children[i]->fops->lease, &local->loc,
+                       &local->cont.lease.user_lease, NULL);
+
+            if (!--call_count)
+                break;
         }
+    }
 
-        call_count = local->call_count;
-        frame->local = local;
+    return 0;
+}
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND (frame, afr_finodelk_cbk,
-                                    priv->children[i],
-                                    priv->children[i]->fops->finodelk,
-                                    volume, fd, cmd, flock);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
+int32_t
+afr_lease_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct gf_lease *lease, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int child_index = -1;
 
-        op_ret = 0;
-out:
-        if (op_ret == -1) {
-                AFR_STACK_UNWIND (finodelk, frame, op_ret, op_errno);
-        }
+    local = frame->local;
+    priv = this->private;
+
+    child_index = (long)cookie;
+
+    afr_common_lock_cbk(frame, cookie, this, op_ret, op_errno, xdata);
+    if (op_ret < 0 && op_errno == EAGAIN) {
+        local->op_ret = -1;
+        local->op_errno = EAGAIN;
+
+        afr_lease_unlock(frame, this);
         return 0;
-}
+    }
+
+    if (op_ret == 0) {
+        local->op_ret = 0;
+        local->op_errno = 0;
+        local->cont.lease.locked_nodes[child_index] = 1;
+        local->cont.lease.ret_lease = *lease;
+    }
+
+    child_index++;
+    if (child_index < priv->child_count) {
+        STACK_WIND_COOKIE(frame, afr_lease_cbk, (void *)(long)child_index,
+                          priv->children[child_index],
+                          priv->children[child_index]->fops->lease, &local->loc,
+                          &local->cont.lease.user_lease, xdata);
+    } else if (priv->quorum_count &&
+               !afr_has_quorum(local->cont.lease.locked_nodes, this, NULL)) {
+        local->op_ret = -1;
+        local->op_errno = afr_final_errno(local, priv);
 
+        afr_lease_unlock(frame, this);
+    } else {
+        if (local->op_ret < 0)
+            local->op_errno = afr_final_errno(local, priv);
+        AFR_STACK_UNWIND(lease, frame, local->op_ret, local->op_errno,
+                         &local->cont.lease.ret_lease, NULL);
+    }
 
-int32_t
-afr_entrylk_cbk (call_frame_t *frame, void *cookie,
-                 xlator_t *this, int32_t op_ret, int32_t op_errno)
+    return 0;
+}
 
+int
+afr_lease(call_frame_t *frame, xlator_t *this, loc_t *loc,
+          struct gf_lease *lease, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int32_t op_errno = ENOMEM;
 
-        int call_count = -1;
+    priv = this->private;
 
-        local = frame->local;
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-        LOCK (&frame->lock);
-        {
-                if (op_ret == 0)
-                        local->op_ret = 0;
+    local->op = GF_FOP_LEASE;
+    local->cont.lease.locked_nodes = GF_CALLOC(
+        priv->child_count, sizeof(*local->cont.lease.locked_nodes),
+        gf_afr_mt_char);
 
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
+    if (!local->cont.lease.locked_nodes) {
+        op_errno = ENOMEM;
+        goto out;
+    }
 
-        call_count = afr_frame_return (frame);
+    loc_copy(&local->loc, loc);
+    local->cont.lease.user_lease = *lease;
+    local->cont.lease.ret_lease = *lease;
 
-        if (call_count == 0)
-                AFR_STACK_UNWIND (entrylk, frame, local->op_ret,
-                                  local->op_errno);
+    STACK_WIND_COOKIE(frame, afr_lease_cbk, (void *)(long)0, priv->children[0],
+                      priv->children[0]->fops->lease, loc, lease, xdata);
 
-        return 0;
-}
+    return 0;
+out:
+    AFR_STACK_UNWIND(lease, frame, -1, op_errno, NULL, NULL);
 
+    return 0;
+}
 
-int32_t
-afr_entrylk (call_frame_t *frame, xlator_t *this,
-             const char *volume, loc_t *loc,
-             const char *basename, entrylk_cmd cmd, entrylk_type type)
+int
+afr_ipc_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+            int32_t op_errno, dict_t *xdata)
 {
-        afr_private_t *priv = NULL;
-        afr_local_t *local  = NULL;
+    afr_local_t *local = NULL;
+    int child_index = (long)cookie;
+    int call_count = 0;
+    gf_boolean_t failed = _gf_false;
+    gf_boolean_t succeeded = _gf_false;
+    int i = 0;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    local->replies[child_index].valid = 1;
+    local->replies[child_index].op_ret = op_ret;
+    local->replies[child_index].op_errno = op_errno;
+    if (xdata)
+        local->replies[child_index].xdata = dict_ref(xdata);
+
+    call_count = afr_frame_return(frame);
+    if (call_count)
+        goto out;
+    /* If any of the subvolumes failed with other than ENOTCONN
+     * return error else return success unless all the subvolumes
+     * failed.
+     * TODO: In case of failure, we need to unregister the xattrs
+     * from the other subvolumes where it succeeded (once upcall
+     * fixes the Bz-1371622)*/
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid)
+            continue;
+        if (local->replies[i].op_ret < 0 &&
+            local->replies[i].op_errno != ENOTCONN) {
+            local->op_ret = local->replies[i].op_ret;
+            local->op_errno = local->replies[i].op_errno;
+            if (local->xdata_rsp)
+                dict_unref(local->xdata_rsp);
+            local->xdata_rsp = NULL;
+            if (local->replies[i].xdata) {
+                local->xdata_rsp = dict_ref(local->replies[i].xdata);
+            }
+            failed = _gf_true;
+            break;
+        }
+        if (local->replies[i].op_ret == 0) {
+            succeeded = _gf_true;
+            local->op_ret = 0;
+            local->op_errno = 0;
+            if (!local->xdata_rsp && local->replies[i].xdata) {
+                local->xdata_rsp = dict_ref(local->replies[i].xdata);
+            }
+        }
+    }
 
-        int ret = -1;
+    if (!succeeded && !failed) {
+        local->op_ret = -1;
+        local->op_errno = ENOTCONN;
+    }
+
+    AFR_STACK_UNWIND(ipc, frame, local->op_ret, local->op_errno,
+                     local->xdata_rsp);
 
-        int i = 0;
-        int32_t call_count = 0;
-        int32_t op_ret   = -1;
-        int32_t op_errno = 0;
+out:
+    return 0;
+}
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+int
+afr_ipc(call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    int32_t op_errno = -1;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    int call_cnt = -1;
 
-        priv = this->private;
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
 
-        ALLOC_OR_GOTO (local, afr_local_t, out);
+    if (op != GF_IPC_TARGET_UPCALL)
+        goto wind_default;
 
-        ret = AFR_LOCAL_INIT (local, priv);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
-        }
+    VALIDATE_OR_GOTO(this->private, err);
+    priv = this->private;
 
-        call_count = local->call_count;
-        frame->local = local;
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto err;
 
+    call_cnt = local->call_count;
+
+    if (xdata) {
         for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND (frame, afr_entrylk_cbk,
-                                    priv->children[i],
-                                    priv->children[i]->fops->entrylk,
-                                    volume, loc, basename, cmd, type);
-
-                        if (!--call_count)
-                                break;
-                }
+            if (dict_set_int8(xdata, priv->pending_key[i], 0) < 0)
+                goto err;
         }
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->child_up[i])
+            continue;
+
+        STACK_WIND_COOKIE(frame, afr_ipc_cbk, (void *)(long)i,
+                          priv->children[i], priv->children[i]->fops->ipc, op,
+                          xdata);
+        if (!--call_cnt)
+            break;
+    }
+    return 0;
+
+err:
+    if (op_errno == -1)
+        op_errno = errno;
+    AFR_STACK_UNWIND(ipc, frame, -1, op_errno, NULL);
+
+    return 0;
+
+wind_default:
+    STACK_WIND(frame, default_ipc_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->ipc, op, xdata);
+    return 0;
+}
 
-        op_ret = 0;
-out:
-        if (op_ret == -1) {
-                AFR_STACK_UNWIND (entrylk, frame, op_ret, op_errno);
-        }
+int
+afr_forget(xlator_t *this, inode_t *inode)
+{
+    uint64_t ctx_int = 0;
+    afr_inode_ctx_t *ctx = NULL;
+
+    afr_spb_choice_timeout_cancel(this, inode);
+    inode_ctx_del(inode, this, &ctx_int);
+    if (!ctx_int)
         return 0;
+
+    ctx = (afr_inode_ctx_t *)(uintptr_t)ctx_int;
+    afr_inode_ctx_destroy(ctx);
+    return 0;
 }
 
+int
+afr_priv_dump(xlator_t *this)
+{
+    afr_private_t *priv = NULL;
+    char key_prefix[GF_DUMP_MAX_BUF_LEN];
+    char key[GF_DUMP_MAX_BUF_LEN];
+    int i = 0;
+
+    GF_ASSERT(this);
+    priv = this->private;
+
+    GF_ASSERT(priv);
+    snprintf(key_prefix, GF_DUMP_MAX_BUF_LEN, "%s.%s", this->type, this->name);
+    gf_proc_dump_add_section("%s", key_prefix);
+    gf_proc_dump_write("child_count", "%u", priv->child_count);
+    for (i = 0; i < priv->child_count; i++) {
+        sprintf(key, "child_up[%d]", i);
+        gf_proc_dump_write(key, "%d", priv->child_up[i]);
+        sprintf(key, "pending_key[%d]", i);
+        gf_proc_dump_write(key, "%s", priv->pending_key[i]);
+        sprintf(key, "pending_reads[%d]", i);
+        gf_proc_dump_write(key, "%" PRId64,
+                           GF_ATOMIC_GET(priv->pending_reads[i]));
+        sprintf(key, "child_latency[%d]", i);
+        gf_proc_dump_write(key, "%" PRId64, priv->child_latency[i]);
+        sprintf(key, "halo_child_up[%d]", i);
+        gf_proc_dump_write(key, "%d", priv->halo_child_up[i]);
+    }
+    gf_proc_dump_write("data_self_heal", "%d", priv->data_self_heal);
+    gf_proc_dump_write("metadata_self_heal", "%d", priv->metadata_self_heal);
+    gf_proc_dump_write("entry_self_heal", "%d", priv->entry_self_heal);
+    gf_proc_dump_write("read_child", "%d", priv->read_child);
+    gf_proc_dump_write("wait_count", "%u", priv->wait_count);
+    gf_proc_dump_write("heal-wait-queue-length", "%d", priv->heal_wait_qlen);
+    gf_proc_dump_write("heal-waiters", "%d", priv->heal_waiters);
+    gf_proc_dump_write("background-self-heal-count", "%d",
+                       priv->background_self_heal_count);
+    gf_proc_dump_write("healers", "%d", priv->healers);
+    gf_proc_dump_write("read-hash-mode", "%d", priv->hash_mode);
+    gf_proc_dump_write("use-anonymous-inode", "%d", priv->use_anon_inode);
+    if (priv->quorum_count == AFR_QUORUM_AUTO) {
+        gf_proc_dump_write("quorum-type", "auto");
+    } else if (priv->quorum_count == 0) {
+        gf_proc_dump_write("quorum-type", "none");
+    } else {
+        gf_proc_dump_write("quorum-type", "fixed");
+        gf_proc_dump_write("quorum-count", "%d", priv->quorum_count);
+    }
+    gf_proc_dump_write("up", "%u", afr_has_quorum(priv->child_up, this, NULL));
+    if (priv->thin_arbiter_count) {
+        gf_proc_dump_write("ta_child_up", "%d", priv->ta_child_up);
+        gf_proc_dump_write("ta_bad_child_index", "%d",
+                           priv->ta_bad_child_index);
+        gf_proc_dump_write("ta_notify_dom_lock_offset", "%" PRId64,
+                           priv->ta_notify_dom_lock_offset);
+    }
+
+    return 0;
+}
 
+/**
+ * find_child_index - find the child's index in the array of subvolumes
+ * @this: AFR
+ * @child: child
+ */
 
-int32_t
-afr_fentrylk_cbk (call_frame_t *frame, void *cookie,
-                 xlator_t *this, int32_t op_ret, int32_t op_errno)
+static int
+afr_find_child_index(xlator_t *this, xlator_t *child)
+{
+    afr_private_t *priv = NULL;
+    int child_count = -1;
+    int i = -1;
+
+    priv = this->private;
+    child_count = priv->child_count;
+    if (priv->thin_arbiter_count) {
+        child_count++;
+    }
+
+    for (i = 0; i < child_count; i++) {
+        if ((xlator_t *)child == priv->children[i])
+            break;
+    }
+
+    return i;
+}
 
+int
+__afr_get_up_children_count(afr_private_t *priv)
 {
-        afr_local_t *local = NULL;
+    int up_children = 0;
+    int i = 0;
 
-        int call_count = -1;
+    for (i = 0; i < priv->child_count; i++)
+        if (priv->child_up[i] == 1)
+            up_children++;
 
-        local = frame->local;
+    return up_children;
+}
 
-        LOCK (&frame->lock);
-        {
-                if (op_ret == 0)
-                        local->op_ret = 0;
+static int
+__get_heard_from_all_status(xlator_t *this)
+{
+    afr_private_t *priv = this->private;
+    int i;
 
-                local->op_errno = op_errno;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!priv->last_event[i]) {
+            return 0;
         }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
+    }
+    if (priv->thin_arbiter_count && !priv->ta_child_up) {
+        return 0;
+    }
+    return 1;
+}
 
-        if (call_count == 0)
-                AFR_STACK_UNWIND (fentrylk, frame, local->op_ret,
-                                  local->op_errno);
+glusterfs_event_t
+__afr_transform_event_from_state(xlator_t *this)
+{
+    int i = 0;
+    int up_children = 0;
+    afr_private_t *priv = this->private;
+
+    if (__get_heard_from_all_status(this))
+        /* have_heard_from_all. Let afr_notify() do the propagation. */
+        return GF_EVENT_MAXVAL;
+
+    up_children = __afr_get_up_children_count(priv);
+    /* Treat the children with pending notification, as having sent a
+     * GF_EVENT_CHILD_DOWN. i.e. set the event as GF_EVENT_SOME_DESCENDENT_DOWN,
+     * as done in afr_notify() */
+    for (i = 0; i < priv->child_count; i++) {
+        if (priv->last_event[i])
+            continue;
+        priv->last_event[i] = GF_EVENT_SOME_DESCENDENT_DOWN;
+        priv->child_up[i] = 0;
+    }
+
+    if (up_children)
+        /* We received at least one child up */
+        return GF_EVENT_CHILD_UP;
+    else
+        return GF_EVENT_CHILD_DOWN;
+
+    return GF_EVENT_MAXVAL;
+}
 
-        return 0;
+static void
+afr_notify_cbk(void *data)
+{
+    xlator_t *this = data;
+    afr_private_t *priv = this->private;
+    glusterfs_event_t event = GF_EVENT_MAXVAL;
+    gf_boolean_t propagate = _gf_false;
+
+    LOCK(&priv->lock);
+    {
+        if (!priv->timer) {
+            /*
+             * Either child_up/child_down is already sent to parent.
+             * This is a spurious wake up.
+             */
+            goto unlock;
+        }
+        priv->timer = NULL;
+        event = __afr_transform_event_from_state(this);
+        if (event != GF_EVENT_MAXVAL)
+            propagate = _gf_true;
+    }
+unlock:
+    UNLOCK(&priv->lock);
+    if (propagate)
+        default_notify(this, event, NULL);
 }
 
+static void
+__afr_launch_notify_timer(xlator_t *this, afr_private_t *priv)
+{
+    struct timespec delay = {
+        0,
+    };
+
+    gf_msg_debug(this->name, 0, "Initiating child-down timer");
+    delay.tv_sec = 10;
+    delay.tv_nsec = 0;
+    priv->timer = gf_timer_call_after(this->ctx, delay, afr_notify_cbk, this);
+    if (priv->timer == NULL) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_TIMER_CREATE_FAIL,
+               "Cannot create timer for delayed initialization");
+    }
+}
 
-int32_t
-afr_fentrylk (call_frame_t *frame, xlator_t *this,
-              const char *volume, fd_t *fd,
-              const char *basename, entrylk_cmd cmd, entrylk_type type)
+static int
+find_best_down_child(xlator_t *this)
 {
-        afr_private_t *priv = NULL;
-        afr_local_t *local  = NULL;
+    afr_private_t *priv = NULL;
+    int i = -1;
+    int32_t best_child = -1;
+    int64_t best_latency = INT64_MAX;
+
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!priv->child_up[i] && priv->child_latency[i] >= 0 &&
+            priv->child_latency[i] < best_latency) {
+            best_child = i;
+            best_latency = priv->child_latency[i];
+        }
+    }
+    if (best_child >= 0) {
+        gf_msg_debug(this->name, 0,
+                     "Found best down child (%d) @ %" PRId64 " ms latency",
+                     best_child, best_latency);
+    }
+    return best_child;
+}
 
-        int ret = -1;
+int
+find_worst_up_child(xlator_t *this)
+{
+    afr_private_t *priv = NULL;
+    int i = -1;
+    int32_t worst_child = -1;
+    int64_t worst_latency = INT64_MIN;
+
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (priv->child_up[i] && priv->child_latency[i] >= 0 &&
+            priv->child_latency[i] > worst_latency) {
+            worst_child = i;
+            worst_latency = priv->child_latency[i];
+        }
+    }
+    if (worst_child >= 0) {
+        gf_msg_debug(this->name, 0,
+                     "Found worst up child (%d) @ %" PRId64 " ms latency",
+                     worst_child, worst_latency);
+    }
+    return worst_child;
+}
 
-        int i = 0;
-        int32_t call_count = 0;
-        int32_t op_ret   = -1;
-        int32_t op_errno = 0;
+void
+__afr_handle_ping_event(xlator_t *this, xlator_t *child_xlator, const int idx,
+                        int64_t halo_max_latency_msec, int32_t *event,
+                        int64_t child_latency_msec)
+{
+    afr_private_t *priv = NULL;
+    int up_children = 0;
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+    priv = this->private;
 
-        priv = this->private;
+    priv->child_latency[idx] = child_latency_msec;
+    gf_msg_debug(child_xlator->name, 0, "Client ping @ %" PRId64 " ms",
+                 child_latency_msec);
+    if (priv->shd.iamshd)
+        return;
 
-        ALLOC_OR_GOTO (local, afr_local_t, out);
+    up_children = __afr_get_up_children_count(priv);
 
-        ret = AFR_LOCAL_INIT (local, priv);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
+    if (child_latency_msec > halo_max_latency_msec &&
+        priv->child_up[idx] == 1 && up_children > priv->halo_min_replicas) {
+        if ((up_children - 1) < priv->halo_min_replicas) {
+            gf_log(child_xlator->name, GF_LOG_INFO,
+                   "Overriding halo threshold, "
+                   "min replicas: %d",
+                   priv->halo_min_replicas);
+        } else {
+            gf_log(child_xlator->name, GF_LOG_INFO,
+                   "Child latency (%" PRId64
+                   " ms) "
+                   "exceeds halo threshold (%" PRId64
+                   "), "
+                   "marking child down.",
+                   child_latency_msec, halo_max_latency_msec);
+            if (priv->halo_child_up[idx]) {
+                *event = GF_EVENT_CHILD_DOWN;
+            }
         }
+    } else if (child_latency_msec < halo_max_latency_msec &&
+               priv->child_up[idx] == 0) {
+        if (up_children < priv->halo_max_replicas) {
+            gf_log(child_xlator->name, GF_LOG_INFO,
+                   "Child latency (%" PRId64
+                   " ms) "
+                   "below halo threshold (%" PRId64
+                   "), "
+                   "marking child up.",
+                   child_latency_msec, halo_max_latency_msec);
+            if (priv->halo_child_up[idx]) {
+                *event = GF_EVENT_CHILD_UP;
+            }
+        } else {
+            gf_log(child_xlator->name, GF_LOG_INFO,
+                   "Not marking child %d up, "
+                   "max replicas (%d) reached.",
+                   idx, priv->halo_max_replicas);
+        }
+    }
+}
 
-        call_count = local->call_count;
-        frame->local = local;
+static int64_t
+afr_get_halo_latency(xlator_t *this)
+{
+    afr_private_t *priv = NULL;
+    int64_t halo_max_latency_msec = 0;
+
+    priv = this->private;
+
+    if (priv->shd.iamshd) {
+        halo_max_latency_msec = priv->shd.halo_max_latency_msec;
+    } else if (priv->nfsd.iamnfsd) {
+        halo_max_latency_msec = priv->nfsd.halo_max_latency_msec;
+    } else {
+        halo_max_latency_msec = priv->halo_max_latency_msec;
+    }
+    gf_msg_debug(this->name, 0, "Using halo latency %" PRId64,
+                 halo_max_latency_msec);
+    return halo_max_latency_msec;
+}
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND (frame, afr_fentrylk_cbk,
-                                    priv->children[i],
-                                    priv->children[i]->fops->fentrylk,
-                                    volume, fd, basename, cmd, type);
-
-                        if (!--call_count)
-                                break;
-                }
+void
+__afr_handle_child_up_event(xlator_t *this, xlator_t *child_xlator,
+                            const int idx, int64_t child_latency_msec,
+                            int32_t *event, int32_t *call_psh,
+                            int32_t *up_child)
+{
+    afr_private_t *priv = NULL;
+    int up_children = 0;
+    int worst_up_child = -1;
+    int64_t halo_max_latency_msec = afr_get_halo_latency(this);
+
+    priv = this->private;
+
+    /*
+     * This only really counts if the child was never up
+     * (value = -1) or had been down (value = 0).  See
+     * comment at GF_EVENT_CHILD_DOWN for a more detailed
+     * explanation.
+     */
+    if (priv->child_up[idx] != 1) {
+        priv->event_generation++;
+    }
+    priv->child_up[idx] = 1;
+
+    *call_psh = 1;
+    *up_child = idx;
+    up_children = __afr_get_up_children_count(priv);
+    /*
+     * If this is an _actual_ CHILD_UP event, we
+     * want to set the child_latency to MAX to indicate
+     * the child needs ping data to be available before doing child-up
+     */
+    if (!priv->halo_enabled)
+        goto out;
+
+    if (child_latency_msec < 0) {
+        /*set to INT64_MAX-1 so that it is found for best_down_child*/
+        priv->halo_child_up[idx] = 1;
+        if (priv->child_latency[idx] < 0) {
+            priv->child_latency[idx] = AFR_HALO_MAX_LATENCY;
+        }
+    }
+
+    /*
+     * Handle the edge case where we exceed
+     * halo_min_replicas and we've got a child which is
+     * marked up as it was helping to satisfy the
+     * halo_min_replicas even though it's latency exceeds
+     * halo_max_latency_msec.
+     */
+    if (up_children > priv->halo_min_replicas) {
+        worst_up_child = find_worst_up_child(this);
+        if (worst_up_child >= 0 &&
+            priv->child_latency[worst_up_child] > halo_max_latency_msec) {
+            gf_msg_debug(this->name, 0,
+                         "Marking child %d down, "
+                         "doesn't meet halo threshold (%" PRId64
+                         "), and > "
+                         "halo_min_replicas (%d)",
+                         worst_up_child, halo_max_latency_msec,
+                         priv->halo_min_replicas);
+            priv->child_up[worst_up_child] = 0;
+            up_children--;
         }
+    }
 
-        op_ret = 0;
-out:
-        if (op_ret == -1) {
-                AFR_STACK_UNWIND (fentrylk, frame, op_ret, op_errno);
+    if (up_children > priv->halo_max_replicas && !priv->shd.iamshd) {
+        worst_up_child = find_worst_up_child(this);
+        if (worst_up_child < 0) {
+            worst_up_child = idx;
         }
-        return 0;
+        priv->child_up[worst_up_child] = 0;
+        up_children--;
+        gf_msg_debug(this->name, 0,
+                     "Marking child %d down, "
+                     "up_children (%d) > halo_max_replicas (%d)",
+                     worst_up_child, up_children, priv->halo_max_replicas);
+    }
+out:
+    if (up_children == 1) {
+        gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_SUBVOL_UP,
+               "Subvolume '%s' came back up; "
+               "going online.",
+               child_xlator->name);
+        gf_event(EVENT_AFR_SUBVOL_UP, "client-pid=%d; subvol=%s",
+                 this->ctx->cmd_args.client_pid, this->name);
+    } else {
+        *event = GF_EVENT_SOME_DESCENDENT_UP;
+    }
+
+    priv->last_event[idx] = *event;
 }
 
-int32_t
-afr_statfs_cbk (call_frame_t *frame, void *cookie,
-                xlator_t *this, int32_t op_ret, int32_t op_errno,
-                struct statvfs *statvfs)
+void
+__afr_handle_child_down_event(xlator_t *this, xlator_t *child_xlator, int idx,
+                              int64_t child_latency_msec, int32_t *event,
+                              int32_t *call_psh, int32_t *up_child)
 {
-        afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    int up_children = 0;
+    int down_children = 0;
+    int best_down_child = -1;
+
+    priv = this->private;
+
+    /*
+     * If a brick is down when we start, we'll get a
+     * CHILD_DOWN to indicate its initial state.  There
+     * was never a CHILD_UP in this case, so if we
+     * increment "down_count" the difference between than
+     * and "up_count" will no longer be the number of
+     * children that are currently up.  This has serious
+     * implications e.g. for quorum enforcement, so we
+     * don't increment these values unless the event
+     * represents an actual state transition between "up"
+     * (value = 1) and anything else.
+     */
+    if (priv->child_up[idx] == 1) {
+        priv->event_generation++;
+    }
+
+    /*
+     * If this is an _actual_ CHILD_DOWN event, we
+     * want to set the child_latency to < 0 to indicate
+     * the child is really disconnected.
+     */
+    if (child_latency_msec < 0) {
+        priv->child_latency[idx] = child_latency_msec;
+        priv->halo_child_up[idx] = 0;
+    }
+    priv->child_up[idx] = 0;
+
+    up_children = __afr_get_up_children_count(priv);
+    /*
+     * Handle the edge case where we need to find the
+     * next best child (to mark up) as marking this child
+     * down would cause us to fall below halo_min_replicas.
+     * We will also force the SHD to heal this child _now_
+     * as we want it to be up to date if we are going to
+     * begin using it synchronously.
+     */
+    if (priv->halo_enabled && up_children < priv->halo_min_replicas) {
+        best_down_child = find_best_down_child(this);
+        if (best_down_child >= 0) {
+            gf_msg_debug(this->name, 0,
+                         "Swapping out child %d for "
+                         "child %d to satisfy halo_min_replicas (%d).",
+                         idx, best_down_child, priv->halo_min_replicas);
+            priv->child_up[best_down_child] = 1;
+            *call_psh = 1;
+            *up_child = best_down_child;
+        }
+    }
+    for (i = 0; i < priv->child_count; i++)
+        if (priv->child_up[i] == 0)
+            down_children++;
+    if (down_children == priv->child_count) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SUBVOLS_DOWN,
+               "All subvolumes are down. Going "
+               "offline until at least one of them "
+               "comes back up.");
+        gf_event(EVENT_AFR_SUBVOLS_DOWN, "client-pid=%d; subvol=%s",
+                 this->ctx->cmd_args.client_pid, this->name);
+    } else {
+        *event = GF_EVENT_SOME_DESCENDENT_DOWN;
+    }
+    priv->last_event[idx] = *event;
+}
 
-        int call_count = 0;
+void
+afr_ta_lock_release_synctask(xlator_t *this)
+{
+    call_frame_t *ta_frame = NULL;
+    int ret = 0;
 
-        LOCK (&frame->lock);
-        {
-                local = frame->local;
-
-                if (op_ret == 0) {
-                        local->op_ret   = op_ret;
-
-                        if (local->cont.statfs.buf_set) {
-                                if (statvfs->f_bavail < local->cont.statfs.buf.f_bavail)
-                                        local->cont.statfs.buf = *statvfs;
-                        } else {
-                                local->cont.statfs.buf = *statvfs;
-                                local->cont.statfs.buf_set = 1;
-                        }
-                }
+    ta_frame = afr_ta_frame_create(this);
+    if (!ta_frame) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_THIN_ARB,
+               "Failed to create ta_frame");
+        return;
+    }
+
+    ret = synctask_new(this->ctx->env, afr_release_notify_lock_for_ta,
+                       afr_ta_lock_release_done, ta_frame, this);
+    if (ret) {
+        STACK_DESTROY(ta_frame->root);
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_THIN_ARB,
+               "Failed to release "
+               "AFR_TA_DOM_NOTIFY lock.");
+    }
+}
 
-                if (op_ret == -1)
-                        local->op_errno = op_errno;
+static void
+afr_handle_inodelk_contention(xlator_t *this, struct gf_upcall *upcall)
+{
+    struct gf_upcall_inodelk_contention *lc = NULL;
+    unsigned int inmem_count = 0;
+    unsigned int onwire_count = 0;
+    afr_private_t *priv = this->private;
 
-        }
-        UNLOCK (&frame->lock);
+    lc = upcall->data;
 
-        call_count = afr_frame_return (frame);
+    if (strcmp(lc->domain, AFR_TA_DOM_NOTIFY) != 0)
+        return;
 
-        if (call_count == 0)
-                AFR_STACK_UNWIND (statfs, frame, local->op_ret, local->op_errno,
-                                  &local->cont.statfs.buf);
+    if (priv->shd.iamshd) {
+        /* shd should ignore AFR_TA_DOM_NOTIFY release requests. */
+        return;
+    }
+    LOCK(&priv->lock);
+    {
+        if (priv->release_ta_notify_dom_lock == _gf_true) {
+            /* Ignore multiple release requests from shds.*/
+            UNLOCK(&priv->lock);
+            return;
+        }
+        priv->release_ta_notify_dom_lock = _gf_true;
+        inmem_count = priv->ta_in_mem_txn_count;
+        onwire_count = priv->ta_on_wire_txn_count;
+    }
+    UNLOCK(&priv->lock);
+    if (inmem_count || onwire_count)
+        /* lock release will happen in txn code path after
+         * in-memory or on-wire txns are over.*/
+        return;
 
-        return 0;
+    afr_ta_lock_release_synctask(this);
 }
 
+static void
+afr_handle_upcall_event(xlator_t *this, struct gf_upcall *upcall)
+{
+    struct gf_upcall_cache_invalidation *up_ci = NULL;
+    afr_private_t *priv = this->private;
+    inode_t *inode = NULL;
+    inode_table_t *itable = NULL;
+    int i = 0;
+
+    switch (upcall->event_type) {
+        case GF_UPCALL_INODELK_CONTENTION:
+            afr_handle_inodelk_contention(this, upcall);
+            break;
+        case GF_UPCALL_CACHE_INVALIDATION:
+            up_ci = (struct gf_upcall_cache_invalidation *)upcall->data;
+
+            /* Since md-cache will be aggressively filtering
+             * lookups, the stale read issue will be more
+             * pronounced. Hence when a pending xattr is set notify
+             * all the md-cache clients to invalidate the existing
+             * stat cache and send the lookup next time */
+            if (!up_ci->dict)
+                break;
+            for (i = 0; i < priv->child_count; i++) {
+                if (!dict_get(up_ci->dict, priv->pending_key[i]))
+                    continue;
+                up_ci->flags |= UP_INVAL_ATTR;
+                itable = ((xlator_t *)this->graph->top)->itable;
+                /*Internal processes may not have itable for
+                 *top xlator*/
+                if (itable)
+                    inode = inode_find(itable, upcall->gfid);
+                if (inode)
+                    afr_inode_need_refresh_set(inode, this);
+                break;
+            }
+            break;
+        default:
+            break;
+    }
+}
 
 int32_t
-afr_statfs (call_frame_t *frame, xlator_t *this,
-            loc_t *loc)
+afr_notify(xlator_t *this, int32_t event, void *data, void *data2)
 {
-        afr_private_t *  priv        = NULL;
-        int              child_count = 0;
-        afr_local_t   *  local       = NULL;
-        int              i           = 0;
-
-        int ret = -1;
-        int              call_count = 0;
-        int32_t          op_ret      = -1;
-        int32_t          op_errno    = 0;
+    afr_private_t *priv = NULL;
+    xlator_t *child_xlator = NULL;
+    int i = -1;
+    int propagate = 0;
+    int had_heard_from_all = 0;
+    int have_heard_from_all = 0;
+    int idx = -1;
+    int ret = -1;
+    int call_psh = 0;
+    int up_child = -1;
+    dict_t *input = NULL;
+    dict_t *output = NULL;
+    gf_boolean_t had_quorum = _gf_false;
+    gf_boolean_t has_quorum = _gf_false;
+    int64_t halo_max_latency_msec = 0;
+    int64_t child_latency_msec = -1;
+
+    child_xlator = (xlator_t *)data;
+
+    priv = this->private;
+
+    if (!priv)
+        return 0;
 
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
-        VALIDATE_OR_GOTO (loc, out);
+    /*
+     * We need to reset this in case children come up in "staggered"
+     * fashion, so that we discover a late-arriving local subvolume.  Note
+     * that we could end up issuing N lookups to the first subvolume, and
+     * O(N^2) overall, but N is small for AFR so it shouldn't be an issue.
+     */
+    priv->did_discovery = _gf_false;
+
+    /* parent xlators don't need to know about every child_up, child_down
+     * because of afr ha. If all subvolumes go down, child_down has
+     * to be triggered. In that state when 1 subvolume comes up child_up
+     * needs to be triggered. dht optimizes revalidate lookup by sending
+     * it only to one of its subvolumes. When child up/down happens
+     * for afr's subvolumes dht should be notified by child_modified. The
+     * subsequent revalidate lookup happens on all the dht's subvolumes
+     * which triggers afr self-heals if any.
+     */
+    idx = afr_find_child_index(this, child_xlator);
+    if (idx < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_INVALID_CHILD_UP,
+               "Received child_up from invalid subvolume");
+        goto out;
+    }
+
+    had_quorum = priv->quorum_count &&
+                 afr_has_quorum(priv->child_up, this, NULL);
+    if (event == GF_EVENT_CHILD_PING) {
+        child_latency_msec = (int64_t)(uintptr_t)data2;
+        if (priv->halo_enabled) {
+            halo_max_latency_msec = afr_get_halo_latency(this);
+
+            /* Calculates the child latency and sets event
+             */
+            LOCK(&priv->lock);
+            {
+                __afr_handle_ping_event(this, child_xlator, idx,
+                                        halo_max_latency_msec, &event,
+                                        child_latency_msec);
+            }
+            UNLOCK(&priv->lock);
+        } else {
+            LOCK(&priv->lock);
+            {
+                priv->child_latency[idx] = child_latency_msec;
+            }
+            UNLOCK(&priv->lock);
+        }
+    }
 
-        priv = this->private;
-        child_count = priv->child_count;
+    if (event == GF_EVENT_CHILD_PING) {
+        /* This is the only xlator that handles PING, no reason to
+         * propagate.
+         */
+        goto out;
+    }
 
-        ALLOC_OR_GOTO (local, afr_local_t, out);
+    if (event == GF_EVENT_TRANSLATOR_OP) {
+        LOCK(&priv->lock);
+        {
+            had_heard_from_all = __get_heard_from_all_status(this);
+        }
+        UNLOCK(&priv->lock);
 
-        ret = AFR_LOCAL_INIT (local, priv);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
+        if (!had_heard_from_all) {
+            ret = -1;
+        } else {
+            input = data;
+            output = data2;
+            ret = afr_xl_op(this, input, output);
         }
+        goto out;
+    }
+
+    if (event == GF_EVENT_UPCALL) {
+        afr_handle_upcall_event(this, data);
+    }
 
-        frame->local = local;
-        call_count = local->call_count;
+    LOCK(&priv->lock);
+    {
+        had_heard_from_all = __get_heard_from_all_status(this);
+        switch (event) {
+            case GF_EVENT_PARENT_UP:
+                __afr_launch_notify_timer(this, priv);
+                propagate = 1;
+                break;
+            case GF_EVENT_CHILD_UP:
+                if (priv->thin_arbiter_count &&
+                    (idx == AFR_CHILD_THIN_ARBITER)) {
+                    priv->ta_child_up = 1;
+                    priv->ta_event_gen++;
+                    break;
+                }
+                __afr_handle_child_up_event(this, child_xlator, idx,
+                                            child_latency_msec, &event,
+                                            &call_psh, &up_child);
+                __afr_lock_heal_synctask(this, priv, idx);
+                break;
 
-        for (i = 0; i < child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND (frame, afr_statfs_cbk,
-                                    priv->children[i],
-                                    priv->children[i]->fops->statfs,
-                                    loc);
-                        if (!--call_count)
-                                break;
+            case GF_EVENT_CHILD_DOWN:
+                if (priv->thin_arbiter_count &&
+                    (idx == AFR_CHILD_THIN_ARBITER)) {
+                    priv->ta_child_up = 0;
+                    priv->ta_event_gen++;
+                    afr_ta_locked_priv_invalidate(priv);
+                    break;
                 }
+                __afr_handle_child_down_event(this, child_xlator, idx,
+                                              child_latency_msec, &event,
+                                              &call_psh, &up_child);
+                __afr_mark_pending_lk_heal(this, priv, idx);
+                break;
+
+            case GF_EVENT_CHILD_CONNECTING:
+                priv->last_event[idx] = event;
+
+                break;
+
+            case GF_EVENT_SOME_DESCENDENT_DOWN:
+                priv->last_event[idx] = event;
+                break;
+            default:
+                propagate = 1;
+                break;
         }
+        have_heard_from_all = __get_heard_from_all_status(this);
+        if (!had_heard_from_all && have_heard_from_all) {
+            if (priv->timer) {
+                gf_timer_call_cancel(this->ctx, priv->timer);
+                priv->timer = NULL;
+            }
+            /* This is the first event which completes aggregation
+               of events from all subvolumes. If at least one subvol
+               had come up, propagate CHILD_UP, but only this time
+            */
+            event = GF_EVENT_CHILD_DOWN;
+            for (i = 0; i < priv->child_count; i++) {
+                if (priv->last_event[i] == GF_EVENT_CHILD_UP) {
+                    event = GF_EVENT_CHILD_UP;
+                    break;
+                }
 
-        op_ret = 0;
-out:
-        if (op_ret == -1) {
-                AFR_STACK_UNWIND (statfs, frame, op_ret, op_errno, NULL);
+                if (priv->last_event[i] == GF_EVENT_CHILD_CONNECTING) {
+                    event = GF_EVENT_CHILD_CONNECTING;
+                    /* continue to check other events for CHILD_UP */
+                }
+            }
         }
-        return 0;
+    }
+    UNLOCK(&priv->lock);
+
+    if (priv->quorum_count) {
+        has_quorum = afr_has_quorum(priv->child_up, this, NULL);
+        if (!had_quorum && has_quorum) {
+            gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_QUORUM_MET,
+                   "Client-quorum is met");
+            gf_event(EVENT_AFR_QUORUM_MET, "client-pid=%d; subvol=%s",
+                     this->ctx->cmd_args.client_pid, this->name);
+        }
+        if (had_quorum && !has_quorum) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_QUORUM_FAIL,
+                   "Client-quorum is not met");
+            gf_event(EVENT_AFR_QUORUM_FAIL, "client-pid=%d; subvol=%s",
+                     this->ctx->cmd_args.client_pid, this->name);
+        }
+    }
+
+    /* if all subvols have reported status, no need to hide anything
+       or wait for anything else. Just propagate blindly */
+    if (have_heard_from_all)
+        propagate = 1;
+
+    ret = 0;
+    if (propagate)
+        ret = default_notify(this, event, data);
+
+    if ((!had_heard_from_all) || call_psh) {
+        /* Launch self-heal on all local subvolumes if:
+         * a) We have_heard_from_all for the first time
+         * b) Already heard from everyone, but we now got a child-up
+         *    event.
+         */
+        if (have_heard_from_all) {
+            afr_selfheal_childup(this, priv);
+        }
+    }
+out:
+    return ret;
 }
 
-
-int32_t
-afr_lk_unlock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct gf_flock *lock)
+int
+afr_local_init(afr_local_t *local, afr_private_t *priv, int32_t *op_errno)
 {
-        afr_local_t * local = NULL;
+    int __ret = -1;
+    local->op_ret = -1;
+    local->op_errno = EUCLEAN;
+
+    __ret = syncbarrier_init(&local->barrier);
+    if (__ret) {
+        if (op_errno)
+            *op_errno = __ret;
+        goto out;
+    }
+
+    local->child_up = GF_MALLOC(priv->child_count * sizeof(*local->child_up),
+                                gf_afr_mt_char);
+    if (!local->child_up) {
+        if (op_errno)
+            *op_errno = ENOMEM;
+        goto out;
+    }
+
+    memcpy(local->child_up, priv->child_up,
+           sizeof(*local->child_up) * priv->child_count);
+    local->call_count = AFR_COUNT(local->child_up, priv->child_count);
+    if (local->call_count == 0) {
+        gf_msg(THIS->name, GF_LOG_INFO, 0, AFR_MSG_SUBVOLS_DOWN,
+               "no subvolumes up");
+        if (op_errno)
+            *op_errno = ENOTCONN;
+        goto out;
+    }
+
+    local->event_generation = priv->event_generation;
+
+    local->read_attempted = GF_CALLOC(priv->child_count, sizeof(char),
+                                      gf_afr_mt_char);
+    if (!local->read_attempted) {
+        if (op_errno)
+            *op_errno = ENOMEM;
+        goto out;
+    }
+
+    local->readable = GF_CALLOC(priv->child_count, sizeof(char),
+                                gf_afr_mt_char);
+    if (!local->readable) {
+        if (op_errno)
+            *op_errno = ENOMEM;
+        goto out;
+    }
+
+    local->readable2 = GF_CALLOC(priv->child_count, sizeof(char),
+                                 gf_afr_mt_char);
+    if (!local->readable2) {
+        if (op_errno)
+            *op_errno = ENOMEM;
+        goto out;
+    }
+
+    local->read_subvol = -1;
+
+    local->replies = GF_CALLOC(priv->child_count, sizeof(*local->replies),
+                               gf_afr_mt_reply_t);
+    if (!local->replies) {
+        if (op_errno)
+            *op_errno = ENOMEM;
+        goto out;
+    }
+
+    local->need_full_crawl = _gf_false;
+    if (priv->thin_arbiter_count) {
+        local->ta_child_up = priv->ta_child_up;
+        local->ta_failed_subvol = AFR_CHILD_UNKNOWN;
+        local->read_txn_query_child = AFR_CHILD_UNKNOWN;
+        local->ta_event_gen = priv->ta_event_gen;
+        local->fop_state = TA_SUCCESS;
+    }
+    local->is_new_entry = _gf_false;
+
+    INIT_LIST_HEAD(&local->healer);
+    return 0;
+out:
+    return -1;
+}
 
-        int call_count = -1;
+int
+afr_internal_lock_init(afr_internal_lock_t *lk, size_t child_count)
+{
+    int ret = -ENOMEM;
 
-        local = frame->local;
-        call_count = afr_frame_return (frame);
+    lk->lower_locked_nodes = GF_CALLOC(sizeof(*lk->lower_locked_nodes),
+                                       child_count, gf_afr_mt_char);
+    if (NULL == lk->lower_locked_nodes)
+        goto out;
 
-        if (call_count == 0)
-                AFR_STACK_UNWIND (lk, frame, local->op_ret, local->op_errno,
-                                  lock);
+    lk->lock_op_ret = -1;
+    lk->lock_op_errno = EUCLEAN;
 
-        return 0;
+    ret = 0;
+out:
+    return ret;
 }
 
-
-int32_t
-afr_lk_unlock (call_frame_t *frame, xlator_t *this)
+void
+afr_matrix_cleanup(int32_t **matrix, unsigned int m)
 {
-        afr_local_t   * local = NULL;
-        afr_private_t * priv  = NULL;
+    int i = 0;
 
-        int i;
-        int call_count = 0;
+    if (!matrix)
+        goto out;
+    for (i = 0; i < m; i++) {
+        GF_FREE(matrix[i]);
+    }
 
-        local = frame->local;
-        priv  = this->private;
+    GF_FREE(matrix);
+out:
+    return;
+}
 
-        call_count = afr_locked_nodes_count (local->cont.lk.locked_nodes,
-                                             priv->child_count);
+int32_t **
+afr_matrix_create(unsigned int m, unsigned int n)
+{
+    int32_t **matrix = NULL;
+    int i = 0;
+
+    matrix = GF_CALLOC(sizeof(*matrix), m, gf_afr_mt_int32_t);
+    if (!matrix)
+        goto out;
+
+    for (i = 0; i < m; i++) {
+        matrix[i] = GF_CALLOC(sizeof(*matrix[i]), n, gf_afr_mt_int32_t);
+        if (!matrix[i])
+            goto out;
+    }
+    return matrix;
+out:
+    afr_matrix_cleanup(matrix, m);
+    return NULL;
+}
 
-        if (call_count == 0) {
-                AFR_STACK_UNWIND (lk, frame, local->op_ret, local->op_errno,
-                                  &local->cont.lk.ret_flock);
-                return 0;
-        }
+int
+afr_transaction_local_init(afr_local_t *local, xlator_t *this)
+{
+    int ret = -ENOMEM;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+    INIT_LIST_HEAD(&local->transaction.wait_list);
+    INIT_LIST_HEAD(&local->transaction.owner_list);
+    INIT_LIST_HEAD(&local->ta_waitq);
+    INIT_LIST_HEAD(&local->ta_onwireq);
+    ret = afr_internal_lock_init(&local->internal_lock, priv->child_count);
+    if (ret < 0)
+        goto out;
+
+    ret = -ENOMEM;
+    local->pre_op_compat = priv->pre_op_compat;
+
+    local->transaction.pre_op = GF_CALLOC(sizeof(*local->transaction.pre_op),
+                                          priv->child_count, gf_afr_mt_char);
+    if (!local->transaction.pre_op)
+        goto out;
+
+    local->transaction.changelog_xdata = GF_CALLOC(
+        sizeof(*local->transaction.changelog_xdata), priv->child_count,
+        gf_afr_mt_dict_t);
+    if (!local->transaction.changelog_xdata)
+        goto out;
+
+    if (priv->arbiter_count == 1) {
+        local->transaction.pre_op_sources = GF_CALLOC(
+            sizeof(*local->transaction.pre_op_sources), priv->child_count,
+            gf_afr_mt_char);
+        if (!local->transaction.pre_op_sources)
+            goto out;
+    }
+
+    local->transaction.failed_subvols = GF_CALLOC(
+        sizeof(*local->transaction.failed_subvols), priv->child_count,
+        gf_afr_mt_char);
+    if (!local->transaction.failed_subvols)
+        goto out;
+
+    local->pending = afr_matrix_create(priv->child_count, AFR_NUM_CHANGE_LOGS);
+    if (!local->pending)
+        goto out;
+
+    ret = 0;
+out:
+    return ret;
+}
 
-        local->call_count = call_count;
+void
+afr_set_low_priority(call_frame_t *frame)
+{
+    frame->root->pid = LOW_PRIO_PROC_PID;
+}
+
+void
+afr_priv_destroy(afr_private_t *priv)
+{
+    int i = 0;
+    int child_count = -1;
 
-        local->cont.lk.user_flock.l_type = F_UNLCK;
+    if (!priv)
+        goto out;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->cont.lk.locked_nodes[i]) {
-                        STACK_WIND (frame, afr_lk_unlock_cbk,
-                                    priv->children[i],
-                                    priv->children[i]->fops->lk,
-                                    local->fd, F_SETLK,
-                                    &local->cont.lk.user_flock);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
+    GF_FREE(priv->sh_domain);
+    GF_FREE(priv->last_event);
 
-        return 0;
+    child_count = priv->child_count;
+    if (priv->thin_arbiter_count) {
+        child_count++;
+    }
+    if (priv->pending_key) {
+        for (i = 0; i < child_count; i++)
+            GF_FREE(priv->pending_key[i]);
+    }
+
+    GF_FREE(priv->pending_reads);
+    GF_FREE(priv->local);
+    GF_FREE(priv->pending_key);
+    GF_FREE(priv->children);
+    GF_FREE(priv->anon_inode);
+    GF_FREE(priv->child_up);
+    GF_FREE(priv->halo_child_up);
+    GF_FREE(priv->child_latency);
+    LOCK_DESTROY(&priv->lock);
+
+    GF_FREE(priv);
+out:
+    return;
 }
 
+int **
+afr_mark_pending_changelog(afr_private_t *priv, unsigned char *pending,
+                           dict_t *xattr, ia_type_t iat)
+{
+    int i = 0;
+    int **changelog = NULL;
+    int idx = -1;
+    int m_idx = 0;
+    int d_idx = 0;
+    int ret = 0;
+
+    m_idx = afr_index_for_transaction_type(AFR_METADATA_TRANSACTION);
+    d_idx = afr_index_for_transaction_type(AFR_DATA_TRANSACTION);
+
+    idx = afr_index_from_ia_type(iat);
+
+    changelog = afr_matrix_create(priv->child_count, AFR_NUM_CHANGE_LOGS);
+    if (!changelog)
+        goto out;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!pending[i])
+            continue;
+
+        changelog[i][m_idx] = hton32(1);
+        if (idx != -1)
+            changelog[i][idx] = hton32(1);
+        /* If the newentry marking is on a newly created directory,
+         * then mark it with the full-heal indicator.
+         */
+        if ((IA_ISDIR(iat)) && (priv->esh_granular))
+            changelog[i][d_idx] = hton32(1);
+    }
+    ret = afr_set_pending_dict(priv, xattr, changelog);
+    if (ret < 0) {
+        afr_matrix_cleanup(changelog, priv->child_count);
+        return NULL;
+    }
+out:
+    return changelog;
+}
 
-int32_t
-afr_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-            int32_t op_ret, int32_t op_errno, struct gf_flock *lock)
+static dict_t *
+afr_set_heal_info(char *status)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-/*        int            ret  = 0; */
+    dict_t *dict = NULL;
+    int ret = -1;
+
+    dict = dict_new();
+    if (!dict) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    ret = dict_set_dynstr_sizen(dict, "heal-info", status);
+    if (ret)
+        gf_msg("", GF_LOG_WARNING, -ret, AFR_MSG_DICT_SET_FAILED,
+               "Failed to set heal-info key to "
+               "%s",
+               status);
+out:
+    /* Any error other than EINVAL, dict_set_dynstr frees status */
+    if (ret == -ENOMEM || ret == -EINVAL) {
+        GF_FREE(status);
+    }
+
+    if (ret && dict) {
+        dict_unref(dict);
+        dict = NULL;
+    }
+    return dict;
+}
 
-        int child_index = -1;
+static gf_boolean_t
+afr_is_dirty_count_non_unary_for_txn(xlator_t *this, struct afr_reply *replies,
+                                     afr_transaction_type type)
+{
+    afr_private_t *priv = this->private;
+    int *dirty = alloca0(priv->child_count * sizeof(int));
+    int i = 0;
 
-        local = frame->local;
-        priv  = this->private;
+    afr_selfheal_extract_xattr(this, replies, type, dirty, NULL);
+    for (i = 0; i < priv->child_count; i++) {
+        if (dirty[i] > 1)
+            return _gf_true;
+    }
 
-        child_index = (long) cookie;
+    return _gf_false;
+}
 
-        if (!child_went_down (op_ret, op_errno) && (op_ret == -1)) {
-                local->op_ret   = -1;
-                local->op_errno = op_errno;
+static gf_boolean_t
+afr_is_dirty_count_non_unary(xlator_t *this, struct afr_reply *replies,
+                             ia_type_t ia_type)
+{
+    gf_boolean_t data_chk = _gf_false;
+    gf_boolean_t mdata_chk = _gf_false;
+    gf_boolean_t entry_chk = _gf_false;
+
+    switch (ia_type) {
+        case IA_IFDIR:
+            mdata_chk = _gf_true;
+            entry_chk = _gf_true;
+            break;
+        case IA_IFREG:
+            mdata_chk = _gf_true;
+            data_chk = _gf_true;
+            break;
+        default:
+            /*IA_IFBLK, IA_IFCHR, IA_IFLNK, IA_IFIFO, IA_IFSOCK*/
+            mdata_chk = _gf_true;
+            break;
+    }
+
+    if (data_chk && afr_is_dirty_count_non_unary_for_txn(
+                        this, replies, AFR_DATA_TRANSACTION)) {
+        return _gf_true;
+    } else if (mdata_chk && afr_is_dirty_count_non_unary_for_txn(
+                                this, replies, AFR_METADATA_TRANSACTION)) {
+        return _gf_true;
+    } else if (entry_chk && afr_is_dirty_count_non_unary_for_txn(
+                                this, replies, AFR_ENTRY_TRANSACTION)) {
+        return _gf_true;
+    }
+
+    return _gf_false;
+}
 
-                afr_lk_unlock (frame, this);
-                return 0;
+static int
+afr_update_heal_status(xlator_t *this, struct afr_reply *replies,
+                       ia_type_t ia_type, gf_boolean_t *esh, gf_boolean_t *dsh,
+                       gf_boolean_t *msh, unsigned char pending)
+{
+    int ret = -1;
+    GF_UNUSED int ret1 = 0;
+    int i = 0;
+    int io_domain_lk_count = 0;
+    int shd_domain_lk_count = 0;
+    afr_private_t *priv = NULL;
+    char *key1 = NULL;
+    char *key2 = NULL;
+
+    priv = this->private;
+    key1 = alloca0(strlen(GLUSTERFS_INODELK_DOM_PREFIX) + 2 +
+                   strlen(this->name));
+    key2 = alloca0(strlen(GLUSTERFS_INODELK_DOM_PREFIX) + 2 +
+                   strlen(priv->sh_domain));
+    sprintf(key1, "%s:%s", GLUSTERFS_INODELK_DOM_PREFIX, this->name);
+    sprintf(key2, "%s:%s", GLUSTERFS_INODELK_DOM_PREFIX, priv->sh_domain);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if ((replies[i].valid != 1) || (replies[i].op_ret != 0))
+            continue;
+        if (!io_domain_lk_count) {
+            ret1 = dict_get_int32(replies[i].xdata, key1, &io_domain_lk_count);
         }
+        if (!shd_domain_lk_count) {
+            ret1 = dict_get_int32(replies[i].xdata, key2, &shd_domain_lk_count);
+        }
+    }
 
-        if (op_ret == 0) {
-                local->op_ret        = 0;
-                local->op_errno      = 0;
-                local->cont.lk.locked_nodes[child_index] = 1;
-                local->cont.lk.ret_flock = *lock;
+    if (!pending) {
+        if ((afr_is_dirty_count_non_unary(this, replies, ia_type)) ||
+            (!io_domain_lk_count)) {
+            /* Needs heal. */
+            ret = 0;
+        } else {
+            /* No heal needed. */
+            *dsh = *esh = *msh = 0;
+        }
+    } else {
+        if (shd_domain_lk_count) {
+            ret = -EAGAIN; /*For 'possibly-healing'. */
+        } else {
+            ret = 0; /*needs heal. Just set a non -ve value so that it is
+                       assumed as the source index.*/
         }
+    }
+    return ret;
+}
 
-        child_index++;
+/*return EIO, EAGAIN or pending*/
+int
+afr_lockless_inspect(call_frame_t *frame, xlator_t *this, uuid_t gfid,
+                     inode_t **inode, gf_boolean_t *entry_selfheal,
+                     gf_boolean_t *data_selfheal,
+                     gf_boolean_t *metadata_selfheal, unsigned char *pending)
+{
+    int ret = -1;
+    int i = 0;
+    afr_private_t *priv = NULL;
+    struct afr_reply *replies = NULL;
+    gf_boolean_t dsh = _gf_false;
+    gf_boolean_t msh = _gf_false;
+    gf_boolean_t esh = _gf_false;
+    unsigned char *sources = NULL;
+    unsigned char *sinks = NULL;
+    unsigned char *valid_on = NULL;
+    uint64_t *witness = NULL;
+
+    priv = this->private;
+    replies = alloca0(sizeof(*replies) * priv->child_count);
+    sources = alloca0(sizeof(*sources) * priv->child_count);
+    sinks = alloca0(sizeof(*sinks) * priv->child_count);
+    witness = alloca0(sizeof(*witness) * priv->child_count);
+    valid_on = alloca0(sizeof(*valid_on) * priv->child_count);
+
+    ret = afr_selfheal_unlocked_inspect(frame, this, gfid, inode, &dsh, &msh,
+                                        &esh, replies);
+    if (ret)
+        goto out;
+    for (i = 0; i < priv->child_count; i++) {
+        if (replies[i].valid && replies[i].op_ret == 0) {
+            valid_on[i] = 1;
+        }
+    }
+    if (msh) {
+        ret = afr_selfheal_find_direction(frame, this, replies,
+                                          AFR_METADATA_TRANSACTION, valid_on,
+                                          sources, sinks, witness, pending);
+        if (*pending & PFLAG_SBRAIN)
+            ret = -EIO;
+        if (ret)
+            goto out;
+    }
+    if (dsh) {
+        ret = afr_selfheal_find_direction(frame, this, replies,
+                                          AFR_DATA_TRANSACTION, valid_on,
+                                          sources, sinks, witness, pending);
+        if (*pending & PFLAG_SBRAIN)
+            ret = -EIO;
+        if (ret)
+            goto out;
+    }
+    if (esh) {
+        ret = afr_selfheal_find_direction(frame, this, replies,
+                                          AFR_ENTRY_TRANSACTION, valid_on,
+                                          sources, sinks, witness, pending);
+        if (*pending & PFLAG_SBRAIN)
+            ret = -EIO;
+        if (ret)
+            goto out;
+    }
 
-        if (child_index < priv->child_count) {
-                STACK_WIND_COOKIE (frame, afr_lk_cbk, (void *) (long) child_index,
-                                   priv->children[child_index],
-                                   priv->children[child_index]->fops->lk,
-                                   local->fd, local->cont.lk.cmd,
-                                   &local->cont.lk.user_flock);
-        } else if (local->op_ret == -1) {
-                /* all nodes have gone down */
+    ret = afr_update_heal_status(this, replies, (*inode)->ia_type, &esh, &dsh,
+                                 &msh, *pending);
+out:
+    *data_selfheal = dsh;
+    *entry_selfheal = esh;
+    *metadata_selfheal = msh;
+    if (replies)
+        afr_replies_wipe(replies, priv->child_count);
+    return ret;
+}
 
-                AFR_STACK_UNWIND (lk, frame, -1, ENOTCONN,
-                                  &local->cont.lk.ret_flock);
+int
+afr_get_heal_info(call_frame_t *frame, xlator_t *this, loc_t *loc)
+{
+    gf_boolean_t data_selfheal = _gf_false;
+    gf_boolean_t metadata_selfheal = _gf_false;
+    gf_boolean_t entry_selfheal = _gf_false;
+    unsigned char pending = 0;
+    dict_t *dict = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
+    inode_t *inode = NULL;
+    char *substr = NULL;
+    char *status = NULL;
+    call_frame_t *heal_frame = NULL;
+    afr_local_t *heal_local = NULL;
+
+    /*Use frame with lk-owner set*/
+    heal_frame = afr_frame_create(frame->this, &op_errno);
+    if (!heal_frame) {
+        ret = -1;
+        goto out;
+    }
+    heal_local = heal_frame->local;
+    heal_frame->local = frame->local;
+
+    ret = afr_lockless_inspect(heal_frame, this, loc->gfid, &inode,
+                               &entry_selfheal, &data_selfheal,
+                               &metadata_selfheal, &pending);
+
+    if (ret == -ENOMEM) {
+        ret = -1;
+        goto out;
+    }
+
+    if (pending & PFLAG_PENDING) {
+        gf_asprintf(&substr, "-pending");
+        if (!substr)
+            goto out;
+    }
+
+    if (ret == -EIO) {
+        ret = gf_asprintf(&status, "split-brain%s", substr ? substr : "");
+        if (ret < 0) {
+            goto out;
+        }
+        dict = afr_set_heal_info(status);
+        if (!dict) {
+            ret = -1;
+            goto out;
+        }
+    } else if (ret == -EAGAIN) {
+        ret = gf_asprintf(&status, "possibly-healing%s", substr ? substr : "");
+        if (ret < 0) {
+            goto out;
+        }
+        dict = afr_set_heal_info(status);
+        if (!dict) {
+            ret = -1;
+            goto out;
+        }
+    } else if (ret >= 0) {
+        /* value of ret = source index
+         * so ret >= 0 and at least one of the 3 booleans set to
+         * true means a source is identified; heal is required.
+         */
+        if (!data_selfheal && !entry_selfheal && !metadata_selfheal) {
+            status = gf_strdup("no-heal");
+            if (!status) {
+                ret = -1;
+                goto out;
+            }
+            dict = afr_set_heal_info(status);
+            if (!dict) {
+                ret = -1;
+                goto out;
+            }
         } else {
-                /* locking has succeeded on all nodes that are up */
+            ret = gf_asprintf(&status, "heal%s", substr ? substr : "");
+            if (ret < 0) {
+                goto out;
+            }
+            dict = afr_set_heal_info(status);
+            if (!dict) {
+                ret = -1;
+                goto out;
+            }
+        }
+    } else if (ret < 0) {
+        /* Apart from above checked -ve ret values, there are
+         * other possible ret values like ENOTCONN
+         * (returned when number of valid replies received are
+         * less than 2)
+         * in which case heal is required when one of the
+         * selfheal booleans is set.
+         */
+        if (data_selfheal || entry_selfheal || metadata_selfheal) {
+            ret = gf_asprintf(&status, "heal%s", substr ? substr : "");
+            if (ret < 0) {
+                goto out;
+            }
+            dict = afr_set_heal_info(status);
+            if (!dict) {
+                ret = -1;
+                goto out;
+            }
+        }
+    }
 
-                /* temporarily
-                ret = afr_mark_locked_nodes (this, local->fd,
-                                             local->cont.lk.locked_nodes);
-                if (ret)
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "Could not save locked nodes info in fdctx");
+    ret = 0;
+    op_errno = 0;
 
-                ret = afr_save_locked_fd (this, local->fd);
-                if (ret)
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "Could not save locked fd");
+out:
+    if (heal_frame) {
+        heal_frame->local = heal_local;
+        AFR_STACK_DESTROY(heal_frame);
+    }
+    AFR_STACK_UNWIND(getxattr, frame, ret, op_errno, dict, NULL);
+    if (dict)
+        dict_unref(dict);
+    if (inode)
+        inode_unref(inode);
+    GF_FREE(substr);
+    return ret;
+}
 
-                */
-		AFR_STACK_UNWIND (lk, frame, local->op_ret, local->op_errno,
-                                  &local->cont.lk.ret_flock);
-        }
+int
+_afr_is_split_brain(call_frame_t *frame, xlator_t *this,
+                    struct afr_reply *replies, afr_transaction_type type,
+                    gf_boolean_t *spb)
+{
+    afr_private_t *priv = NULL;
+    uint64_t *witness = NULL;
+    unsigned char *sources = NULL;
+    unsigned char *sinks = NULL;
+    int sources_count = 0;
+    int ret = 0;
+
+    priv = this->private;
+
+    sources = alloca0(priv->child_count);
+    sinks = alloca0(priv->child_count);
+    witness = alloca0(priv->child_count * sizeof(*witness));
+
+    ret = afr_selfheal_find_direction(frame, this, replies, type,
+                                      priv->child_up, sources, sinks, witness,
+                                      NULL);
+    if (ret)
+        return ret;
 
-        return 0;
-}
+    sources_count = AFR_COUNT(sources, priv->child_count);
+    if (!sources_count)
+        *spb = _gf_true;
 
+    return ret;
+}
 
 int
-afr_lk (call_frame_t *frame, xlator_t *this,
-        fd_t *fd, int32_t cmd,
-        struct gf_flock *flock)
+afr_is_split_brain(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                   uuid_t gfid, gf_boolean_t *d_spb, gf_boolean_t *m_spb)
 {
-        afr_private_t *priv = NULL;
-        afr_local_t *local = NULL;
+    int ret = -1;
+    afr_private_t *priv = NULL;
+    struct afr_reply *replies = NULL;
 
-        int i = 0;
+    priv = this->private;
 
-        int32_t op_ret   = -1;
-        int32_t op_errno = 0;
+    replies = alloca0(sizeof(*replies) * priv->child_count);
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+    ret = afr_selfheal_unlocked_discover(frame, inode, gfid, replies);
+    if (ret)
+        goto out;
 
-        priv = this->private;
+    if (!afr_can_decide_split_brain_source_sinks(replies, priv->child_count)) {
+        ret = -EAGAIN;
+        goto out;
+    }
 
-        ALLOC_OR_GOTO (local, afr_local_t, out);
-        AFR_LOCAL_INIT (local, priv);
+    ret = _afr_is_split_brain(frame, this, replies, AFR_DATA_TRANSACTION,
+                              d_spb);
+    if (ret)
+        goto out;
 
-        frame->local  = local;
+    ret = _afr_is_split_brain(frame, this, replies, AFR_METADATA_TRANSACTION,
+                              m_spb);
+out:
+    if (replies) {
+        afr_replies_wipe(replies, priv->child_count);
+        replies = NULL;
+    }
+    return ret;
+}
 
-        local->cont.lk.locked_nodes = GF_CALLOC (priv->child_count,
-                                              sizeof (*local->cont.lk.locked_nodes),
-                                              gf_afr_mt_char);
+int
+afr_get_split_brain_status_cbk(int ret, call_frame_t *frame, void *opaque)
+{
+    GF_FREE(opaque);
+    return 0;
+}
 
-        if (!local->cont.lk.locked_nodes) {
-                gf_log (this->name, GF_LOG_ERROR, "Out of memory");
-                op_errno = ENOMEM;
-                goto out;
+int
+afr_get_split_brain_status(void *opaque)
+{
+    gf_boolean_t d_spb = _gf_false;
+    gf_boolean_t m_spb = _gf_false;
+    int ret = -1;
+    int op_errno = 0;
+    int i = 0;
+    char *choices = NULL;
+    char *status = NULL;
+    dict_t *dict = NULL;
+    inode_t *inode = NULL;
+    afr_private_t *priv = NULL;
+    xlator_t **children = NULL;
+    call_frame_t *frame = NULL;
+    xlator_t *this = NULL;
+    loc_t *loc = NULL;
+    afr_spb_status_t *data = NULL;
+
+    data = opaque;
+    frame = data->frame;
+    this = frame->this;
+    loc = data->loc;
+    priv = this->private;
+    children = priv->children;
+
+    inode = afr_inode_find(this, loc->gfid);
+    if (!inode)
+        goto out;
+
+    dict = dict_new();
+    if (!dict) {
+        op_errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
+
+    /* Calculation for string length :
+     * (child_count X length of child-name) + SLEN("    Choices :")
+     * child-name consists of :
+     * a) 251 = max characters for volname according to GD_VOLUME_NAME_MAX
+     * b) strlen("-client-00,") assuming 16 replicas
+     */
+    choices = alloca0(priv->child_count * (256 + SLEN("-client-00,")) +
+                      SLEN("    Choices:"));
+
+    ret = afr_is_split_brain(frame, this, inode, loc->gfid, &d_spb, &m_spb);
+    if (ret) {
+        op_errno = -ret;
+        if (ret == -EAGAIN) {
+            ret = dict_set_sizen_str_sizen(dict, GF_AFR_SBRAIN_STATUS,
+                                           SBRAIN_HEAL_NO_GO_MSG);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, -ret,
+                       AFR_MSG_DICT_SET_FAILED,
+                       "Failed to set GF_AFR_SBRAIN_STATUS in dict");
+            }
         }
+        ret = -1;
+        goto out;
+    }
 
-        local->fd            = fd_ref (fd);
-        local->cont.lk.cmd   = cmd;
-        local->cont.lk.user_flock = *flock;
-        local->cont.lk.ret_flock = *flock;
+    if (d_spb || m_spb) {
+        sprintf(choices, "    Choices:");
+        for (i = 0; i < priv->child_count; i++) {
+            strcat(choices, children[i]->name);
+            strcat(choices, ",");
+        }
+        choices[strlen(choices) - 1] = '\0';
 
-        STACK_WIND_COOKIE (frame, afr_lk_cbk, (void *) (long) 0,
-                           priv->children[i],
-                           priv->children[i]->fops->lk,
-                           fd, cmd, flock);
+        ret = gf_asprintf(&status,
+                          "data-split-brain:%s    "
+                          "metadata-split-brain:%s%s",
+                          (d_spb) ? "yes" : "no", (m_spb) ? "yes" : "no",
+                          choices);
+
+        if (-1 == ret) {
+            op_errno = ENOMEM;
+            goto out;
+        }
+        ret = dict_set_dynstr_sizen(dict, GF_AFR_SBRAIN_STATUS, status);
+        if (ret) {
+            op_errno = -ret;
+            ret = -1;
+            goto out;
+        }
+    } else {
+        ret = dict_set_sizen_str_sizen(dict, GF_AFR_SBRAIN_STATUS,
+                                       SFILE_NOT_UNDER_DATA);
+        if (ret) {
+            op_errno = -ret;
+            ret = -1;
+            goto out;
+        }
+    }
 
-        op_ret = 0;
+    ret = 0;
 out:
-        if (op_ret == -1) {
-                AFR_STACK_UNWIND (lk, frame, op_ret, op_errno, NULL);
+    AFR_STACK_UNWIND(getxattr, frame, ret, op_errno, dict, NULL);
+    if (dict)
+        dict_unref(dict);
+    if (inode)
+        inode_unref(inode);
+    return ret;
+}
+
+int32_t
+afr_heal_splitbrain_file(call_frame_t *frame, xlator_t *this, loc_t *loc)
+{
+    int ret = 0;
+    int op_errno = 0;
+    dict_t *dict = NULL;
+    afr_local_t *local = NULL;
+    afr_local_t *heal_local = NULL;
+    call_frame_t *heal_frame = NULL;
+
+    local = frame->local;
+    dict = dict_new();
+    if (!dict) {
+        op_errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
+
+    heal_frame = afr_frame_create(this, &op_errno);
+    if (!heal_frame) {
+        ret = -1;
+        goto out;
+    }
+    heal_local = heal_frame->local;
+    heal_frame->local = frame->local;
+    /*Initiate heal with heal_frame with lk-owner set so that inodelk/entrylk
+     * work correctly*/
+    ret = afr_selfheal_do(heal_frame, this, loc->gfid);
+
+    if (ret == 1 || ret == 2) {
+        ret = dict_set_sizen_str_sizen(dict, "sh-fail-msg",
+                                       SFILE_NOT_IN_SPLIT_BRAIN);
+        if (ret)
+            gf_msg(this->name, GF_LOG_WARNING, -ret, AFR_MSG_DICT_SET_FAILED,
+                   "Failed to set sh-fail-msg in dict");
+        ret = 0;
+        goto out;
+    } else {
+        if (local->xdata_rsp) {
+            /* 'sh-fail-msg' has been set in the dict during self-heal.*/
+            dict_copy(local->xdata_rsp, dict);
+            ret = 0;
+        } else if (ret < 0) {
+            op_errno = -ret;
+            ret = -1;
         }
-        return 0;
+    }
+
+out:
+    if (heal_frame) {
+        heal_frame->local = heal_local;
+        AFR_STACK_DESTROY(heal_frame);
+    }
+    if (local->op == GF_FOP_GETXATTR)
+        AFR_STACK_UNWIND(getxattr, frame, ret, op_errno, dict, NULL);
+    else if (local->op == GF_FOP_SETXATTR)
+        AFR_STACK_UNWIND(setxattr, frame, ret, op_errno, NULL);
+    if (dict)
+        dict_unref(dict);
+    return ret;
 }
 
 int
-afr_priv_dump (xlator_t *this)
+afr_get_child_index_from_name(xlator_t *this, char *name)
 {
-        afr_private_t *priv = NULL;
-        char  key_prefix[GF_DUMP_MAX_BUF_LEN];
-        char  key[GF_DUMP_MAX_BUF_LEN];
-        int   i = 0;
+    afr_private_t *priv = this->private;
+    int index = -1;
+
+    for (index = 0; index < priv->child_count; index++) {
+        if (!strcmp(priv->children[index]->name, name))
+            goto out;
+    }
+    index = -1;
+out:
+    return index;
+}
 
+void
+afr_priv_need_heal_set(afr_private_t *priv, gf_boolean_t need_heal)
+{
+    LOCK(&priv->lock);
+    {
+        priv->need_heal = need_heal;
+    }
+    UNLOCK(&priv->lock);
+}
 
-        GF_ASSERT (this);
-        priv = this->private;
+void
+afr_set_need_heal(xlator_t *this, afr_local_t *local)
+{
+    int i = 0;
+    afr_private_t *priv = this->private;
+    gf_boolean_t need_heal = _gf_false;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->replies[i].valid && local->replies[i].need_heal) {
+            need_heal = _gf_true;
+            break;
+        }
+    }
+    afr_priv_need_heal_set(priv, need_heal);
+    return;
+}
 
-        GF_ASSERT (priv);
-        snprintf(key_prefix, GF_DUMP_MAX_BUF_LEN, "%s.%s", this->type, this->name);
-        gf_proc_dump_add_section(key_prefix);
-        gf_proc_dump_build_key(key, key_prefix, "child_count");
-        gf_proc_dump_write(key, "%u", priv->child_count);
-        gf_proc_dump_build_key(key, key_prefix, "read_child_rr");
-        gf_proc_dump_write(key, "%u", priv->read_child_rr);
-        for (i = 0; i < priv->child_count; i++) {
-                gf_proc_dump_build_key(key, key_prefix, "child_up[%d]", i);
-                gf_proc_dump_write(key, "%d", priv->child_up[i]);
-                gf_proc_dump_build_key(key, key_prefix,
-                                        "pending_key[%d]", i);
-                gf_proc_dump_write(key, "%s", priv->pending_key[i]);
-        }
-        gf_proc_dump_build_key(key, key_prefix, "data_self_heal");
-        gf_proc_dump_write(key, "%d", priv->data_self_heal);
-        gf_proc_dump_build_key(key, key_prefix, "metadata_self_heal");
-        gf_proc_dump_write(key, "%d", priv->metadata_self_heal);
-        gf_proc_dump_build_key(key, key_prefix, "entry_self_heal");
-        gf_proc_dump_write(key, "%d", priv->entry_self_heal);
-        gf_proc_dump_build_key(key, key_prefix, "data_change_log");
-        gf_proc_dump_write(key, "%d", priv->data_change_log);
-        gf_proc_dump_build_key(key, key_prefix, "metadata_change_log");
-        gf_proc_dump_write(key, "%d", priv->metadata_change_log);
-        gf_proc_dump_build_key(key, key_prefix, "entry_change_log");
-        gf_proc_dump_write(key, "%d", priv->entry_change_log);
-        gf_proc_dump_build_key(key, key_prefix, "read_child");
-        gf_proc_dump_write(key, "%d", priv->read_child);
-        gf_proc_dump_build_key(key, key_prefix, "favorite_child");
-        gf_proc_dump_write(key, "%u", priv->favorite_child);
-        gf_proc_dump_build_key(key, key_prefix, "data_lock_server_count");
-        gf_proc_dump_write(key, "%u", priv->data_lock_server_count);
-        gf_proc_dump_build_key(key, key_prefix, "metadata_lock_server_count");
-        gf_proc_dump_write(key, "%u", priv->metadata_lock_server_count);
-        gf_proc_dump_build_key(key, key_prefix, "entry_lock_server_count");
-        gf_proc_dump_write(key, "%u", priv->entry_lock_server_count);
-        gf_proc_dump_build_key(key, key_prefix, "wait_count");
-        gf_proc_dump_write(key, "%u", priv->wait_count);
+gf_boolean_t
+afr_get_need_heal(xlator_t *this)
+{
+    afr_private_t *priv = this->private;
+    gf_boolean_t need_heal = _gf_true;
+
+    LOCK(&priv->lock);
+    {
+        need_heal = priv->need_heal;
+    }
+    UNLOCK(&priv->lock);
+    return need_heal;
+}
 
-        return 0;
+int
+afr_get_msg_id(char *op_type)
+{
+    if (!strcmp(op_type, GF_AFR_REPLACE_BRICK))
+        return AFR_MSG_REPLACE_BRICK_STATUS;
+    else if (!strcmp(op_type, GF_AFR_ADD_BRICK))
+        return AFR_MSG_ADD_BRICK_STATUS;
+    return -1;
 }
 
+int
+afr_fav_child_reset_sink_xattrs_cbk(int ret, call_frame_t *heal_frame,
+                                    void *opaque)
+{
+    call_frame_t *txn_frame = NULL;
+    afr_local_t *local = NULL;
+    afr_local_t *heal_local = NULL;
+    xlator_t *this = NULL;
 
-/**
- * find_child_index - find the child's index in the array of subvolumes
- * @this: AFR
- * @child: child
+    heal_local = heal_frame->local;
+    txn_frame = heal_local->heal_frame;
+    local = txn_frame->local;
+    this = txn_frame->this;
+
+    /* Refresh the inode agan and proceed with the transaction.*/
+    afr_inode_refresh(txn_frame, this, local->inode, NULL, local->refreshfn);
+
+    AFR_STACK_DESTROY(heal_frame);
+
+    return 0;
+}
+
+int
+afr_fav_child_reset_sink_xattrs(void *opaque)
+{
+    call_frame_t *heal_frame = NULL;
+    call_frame_t *txn_frame = NULL;
+    xlator_t *this = NULL;
+    gf_boolean_t d_spb = _gf_false;
+    gf_boolean_t m_spb = _gf_false;
+    afr_local_t *heal_local = NULL;
+    afr_local_t *txn_local = NULL;
+    afr_private_t *priv = NULL;
+    inode_t *inode = NULL;
+    unsigned char *locked_on = NULL;
+    unsigned char *sources = NULL;
+    unsigned char *sinks = NULL;
+    unsigned char *healed_sinks = NULL;
+    unsigned char *undid_pending = NULL;
+    struct afr_reply *locked_replies = NULL;
+    int ret = 0;
+
+    heal_frame = (call_frame_t *)opaque;
+    heal_local = heal_frame->local;
+    txn_frame = heal_local->heal_frame;
+    txn_local = txn_frame->local;
+    this = txn_frame->this;
+    inode = txn_local->inode;
+    priv = this->private;
+    locked_on = alloca0(priv->child_count);
+    sources = alloca0(priv->child_count);
+    sinks = alloca0(priv->child_count);
+    healed_sinks = alloca0(priv->child_count);
+    undid_pending = alloca0(priv->child_count);
+    locked_replies = alloca0(sizeof(*locked_replies) * priv->child_count);
+
+    ret = _afr_is_split_brain(txn_frame, this, txn_local->replies,
+                              AFR_DATA_TRANSACTION, &d_spb);
+
+    ret = _afr_is_split_brain(txn_frame, this, txn_local->replies,
+                              AFR_METADATA_TRANSACTION, &m_spb);
+
+    /* Take appropriate locks and reset sink xattrs. */
+    if (d_spb) {
+        ret = afr_selfheal_inodelk(heal_frame, this, inode, this->name, 0, 0,
+                                   locked_on);
+        {
+            if (ret < priv->child_count)
+                goto data_unlock;
+            ret = __afr_selfheal_data_prepare(
+                heal_frame, this, inode, locked_on, sources, sinks,
+                healed_sinks, undid_pending, locked_replies, NULL);
+        }
+    data_unlock:
+        afr_selfheal_uninodelk(heal_frame, this, inode, this->name, 0, 0,
+                               locked_on);
+    }
+
+    if (m_spb) {
+        memset(locked_on, 0, sizeof(*locked_on) * priv->child_count);
+        memset(undid_pending, 0, sizeof(*undid_pending) * priv->child_count);
+        ret = afr_selfheal_inodelk(heal_frame, this, inode, this->name,
+                                   LLONG_MAX - 1, 0, locked_on);
+        {
+            if (ret < priv->child_count)
+                goto mdata_unlock;
+            ret = __afr_selfheal_metadata_prepare(
+                heal_frame, this, inode, locked_on, sources, sinks,
+                healed_sinks, undid_pending, locked_replies, NULL);
+        }
+    mdata_unlock:
+        afr_selfheal_uninodelk(heal_frame, this, inode, this->name,
+                               LLONG_MAX - 1, 0, locked_on);
+    }
+
+    return ret;
+}
+
+/*
+ * Concatenates the xattrs in local->replies separated by a delimiter.
  */
+int
+afr_serialize_xattrs_with_delimiter(call_frame_t *frame, xlator_t *this,
+                                    char *buf, const char *default_str,
+                                    int32_t *serz_len, char delimiter)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    char *xattr = NULL;
+    int i = 0;
+    int len = 0;
+    int keylen = 0;
+    size_t str_len = 0;
+    int ret = -1;
+
+    priv = this->private;
+    local = frame->local;
+
+    keylen = strlen(local->cont.getxattr.name);
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid || local->replies[i].op_ret) {
+            str_len = strlen(default_str);
+            buf = strncat(buf, default_str, str_len);
+            len += str_len;
+            buf[len++] = delimiter;
+            buf[len] = '\0';
+        } else {
+            ret = dict_get_strn(local->replies[i].xattr,
+                                local->cont.getxattr.name, keylen, &xattr);
+            if (ret) {
+                gf_msg("TEST", GF_LOG_ERROR, -ret, AFR_MSG_DICT_GET_FAILED,
+                       "Failed to get the node_uuid of brick "
+                       "%d",
+                       i);
+                goto out;
+            }
+            str_len = strlen(xattr);
+            buf = strncat(buf, xattr, str_len);
+            len += str_len;
+            buf[len++] = delimiter;
+            buf[len] = '\0';
+        }
+    }
+    buf[--len] = '\0'; /*remove the last delimiter*/
+    if (serz_len)
+        *serz_len = ++len;
+    ret = 0;
 
-static int
-find_child_index (xlator_t *this, xlator_t *child)
+out:
+    return ret;
+}
+
+uint64_t
+afr_write_subvol_get(call_frame_t *frame, xlator_t *this)
 {
-        afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    uint64_t write_subvol = 0;
 
-        int i = -1;
+    local = frame->local;
+    LOCK(&local->inode->lock);
+    write_subvol = local->inode_ctx->write_subvol;
+    UNLOCK(&local->inode->lock);
 
-        priv = this->private;
+    return write_subvol;
+}
 
-        for (i = 0; i < priv->child_count; i++) {
-                if ((xlator_t *) child == priv->children[i])
-                        break;
+int
+afr_write_subvol_set(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    unsigned char *data_accused = NULL;
+    unsigned char *metadata_accused = NULL;
+    unsigned char *data_readable = NULL;
+    unsigned char *metadata_readable = NULL;
+    uint16_t datamap = 0;
+    uint16_t metadatamap = 0;
+    uint64_t val = 0;
+    int event = 0;
+    int i = 0;
+
+    local = frame->local;
+    priv = this->private;
+    data_accused = alloca0(priv->child_count);
+    metadata_accused = alloca0(priv->child_count);
+    data_readable = alloca0(priv->child_count);
+    metadata_readable = alloca0(priv->child_count);
+    event = local->event_generation;
+
+    afr_readables_fill(frame, this, local->inode, data_accused,
+                       metadata_accused, data_readable, metadata_readable,
+                       NULL);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (data_readable[i])
+            datamap |= (1 << i);
+        if (metadata_readable[i])
+            metadatamap |= (1 << i);
+    }
+
+    val = ((uint64_t)metadatamap) | (((uint64_t)datamap) << 16) |
+          (((uint64_t)event) << 32);
+
+    LOCK(&local->inode->lock);
+    {
+        if (local->inode_ctx->write_subvol == 0 &&
+            local->transaction.type == AFR_DATA_TRANSACTION) {
+            local->inode_ctx->write_subvol = val;
         }
+    }
+    UNLOCK(&local->inode->lock);
 
-        return i;
+    return 0;
 }
 
-int32_t
-afr_notify (xlator_t *this, int32_t event,
-            void *data, ...)
+int
+afr_write_subvol_reset(call_frame_t *frame, xlator_t *this)
 {
-        afr_private_t *     priv     = NULL;
-        unsigned char *     child_up = NULL;
+    afr_local_t *local = NULL;
 
-        int i           = -1;
-        int up_children = 0;
-        int down_children = 0;
+    local = frame->local;
+    LOCK(&local->inode->lock);
+    {
+        GF_ASSERT(local->inode_ctx->lock_count > 0);
+        local->inode_ctx->lock_count--;
 
-        priv = this->private;
+        if (!local->inode_ctx->lock_count)
+            local->inode_ctx->write_subvol = 0;
+    }
+    UNLOCK(&local->inode->lock);
 
-        if (!priv)
-                return 0;
+    return 0;
+}
 
-        child_up = priv->child_up;
+int
+afr_set_inode_local(xlator_t *this, afr_local_t *local, inode_t *inode)
+{
+    int ret = 0;
+
+    local->inode = inode_ref(inode);
+    LOCK(&local->inode->lock);
+    {
+        ret = __afr_inode_ctx_get(this, local->inode, &local->inode_ctx);
+    }
+    UNLOCK(&local->inode->lock);
+    if (ret < 0) {
+        gf_msg_callingfn(
+            this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_INODE_CTX_GET_FAILED,
+            "Error getting inode ctx %s", uuid_utoa(local->inode->gfid));
+    }
+    return ret;
+}
 
-        switch (event) {
-        case GF_EVENT_CHILD_UP:
-                i = find_child_index (this, data);
+gf_boolean_t
+afr_ta_is_fop_called_from_synctask(xlator_t *this)
+{
+    struct synctask *task = NULL;
+    gf_lkowner_t tmp_owner = {
+        0,
+    };
 
-                /* temporarily
-                afr_attempt_lock_recovery (this, i);
-                */
+    task = synctask_get();
+    if (!task)
+        return _gf_false;
 
-                child_up[i] = 1;
+    set_lk_owner_from_ptr(&tmp_owner, (void *)this);
 
-                LOCK (&priv->lock);
-                {
-                        priv->up_count++;
-                }
-                UNLOCK (&priv->lock);
+    if (!is_same_lkowner(&tmp_owner, &task->frame->root->lk_owner))
+        return _gf_false;
 
-                /*
-                   if all the children were down, and one child came up,
-                   send notify to parent
-                */
+    return _gf_true;
+}
 
-                for (i = 0; i < priv->child_count; i++)
-                        if (child_up[i] == 1)
-                                up_children++;
+int
+afr_ta_post_op_lock(xlator_t *this, loc_t *loc)
+{
+    int ret = 0;
+    uuid_t gfid = {
+        0,
+    };
+    afr_private_t *priv = this->private;
+    gf_boolean_t locked = _gf_false;
+    struct gf_flock flock1 = {
+        0,
+    };
+    struct gf_flock flock2 = {
+        0,
+    };
+    int32_t cmd = 0;
+
+    /* Clients must take AFR_TA_DOM_NOTIFY lock only when the previous lock
+     * has been released in afr_notify due to upcall notification from shd.
+     */
+    GF_ASSERT(priv->ta_notify_dom_lock_offset == 0);
+
+    if (!priv->shd.iamshd)
+        GF_ASSERT(afr_ta_is_fop_called_from_synctask(this));
+    flock1.l_type = F_WRLCK;
+
+    while (!locked) {
+        if (priv->shd.iamshd) {
+            cmd = F_SETLKW;
+            flock1.l_start = 0;
+            flock1.l_len = 0;
+        } else {
+            cmd = F_SETLK;
+            gf_uuid_generate(gfid);
+            flock1.l_start = gfid_to_ino(gfid);
+            if (flock1.l_start < 0)
+                flock1.l_start = -flock1.l_start;
+            flock1.l_len = 1;
+        }
+        ret = syncop_inodelk(priv->children[THIN_ARBITER_BRICK_INDEX],
+                             AFR_TA_DOM_NOTIFY, loc, cmd, &flock1, NULL, NULL);
+        if (!ret) {
+            locked = _gf_true;
+            priv->ta_notify_dom_lock_offset = flock1.l_start;
+        } else if (ret == -EAGAIN) {
+            continue;
+        } else {
+            gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+                   "Failed to get "
+                   "AFR_TA_DOM_NOTIFY lock on %s.",
+                   loc->name);
+            goto out;
+        }
+    }
+
+    flock2.l_type = F_WRLCK;
+    flock2.l_start = 0;
+    flock2.l_len = 0;
+    ret = syncop_inodelk(priv->children[THIN_ARBITER_BRICK_INDEX],
+                         AFR_TA_DOM_MODIFY, loc, F_SETLKW, &flock2, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed to get AFR_TA_DOM_MODIFY lock on %s.", loc->name);
+        flock1.l_type = F_UNLCK;
+        ret = syncop_inodelk(priv->children[THIN_ARBITER_BRICK_INDEX],
+                             AFR_TA_DOM_NOTIFY, loc, F_SETLK, &flock1, NULL,
+                             NULL);
+    }
+out:
+    return ret;
+}
 
-                if (up_children == 1) {
-                        gf_log (this->name, GF_LOG_NORMAL,
-                                "Subvolume '%s' came back up; "
-                                "going online.", ((xlator_t *)data)->name);
+int
+afr_ta_post_op_unlock(xlator_t *this, loc_t *loc)
+{
+    afr_private_t *priv = this->private;
+    struct gf_flock flock = {
+        0,
+    };
+    int ret = 0;
+
+    if (!priv->shd.iamshd)
+        GF_ASSERT(afr_ta_is_fop_called_from_synctask(this));
+    flock.l_type = F_UNLCK;
+    flock.l_start = 0;
+    flock.l_len = 0;
+
+    ret = syncop_inodelk(priv->children[THIN_ARBITER_BRICK_INDEX],
+                         AFR_TA_DOM_MODIFY, loc, F_SETLK, &flock, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed to unlock AFR_TA_DOM_MODIFY lock.");
+        goto out;
+    }
+
+    if (!priv->shd.iamshd)
+        /* Mounts (clients) will not release the AFR_TA_DOM_NOTIFY lock
+         * in post-op as they use it as a notification mechanism. When
+         * shd sends a lock request on TA during heal, the clients will
+         * receive a lock-contention upcall notification upon which they
+         * will release the AFR_TA_DOM_NOTIFY lock after completing the
+         * in flight I/O.*/
+        goto out;
+
+    ret = syncop_inodelk(priv->children[THIN_ARBITER_BRICK_INDEX],
+                         AFR_TA_DOM_NOTIFY, loc, F_SETLK, &flock, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed to unlock AFR_TA_DOM_NOTIFY lock.");
+    }
+out:
+    return ret;
+}
 
-                        default_notify (this, event, data);
-                }
+call_frame_t *
+afr_ta_frame_create(xlator_t *this)
+{
+    call_frame_t *frame = NULL;
+    void *lk_owner = NULL;
+
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame)
+        return NULL;
+    lk_owner = (void *)this;
+    afr_set_lk_owner(frame, this, lk_owner);
+    return frame;
+}
 
-                break;
+gf_boolean_t
+afr_ta_has_quorum(afr_private_t *priv, afr_local_t *local)
+{
+    int data_count = 0;
 
-        case GF_EVENT_CHILD_DOWN:
-                i = find_child_index (this, data);
+    data_count = AFR_COUNT(local->child_up, priv->child_count);
+    if (data_count == 2) {
+        return _gf_true;
+    } else if (data_count == 1 && local->ta_child_up) {
+        return _gf_true;
+    }
 
-                child_up[i] = 0;
+    return _gf_false;
+}
 
-                LOCK (&priv->lock);
-                {
-                        priv->down_count++;
-                }
-                UNLOCK (&priv->lock);
+static gf_boolean_t
+afr_is_add_replica_mount_lookup_on_root(call_frame_t *frame)
+{
+    afr_local_t *local = NULL;
 
-                /*
-                   if all children are down, and this was the last to go down,
-                   send notify to parent
-                */
+    if (frame->root->pid != GF_CLIENT_PID_ADD_REPLICA_MOUNT)
+        return _gf_false;
 
-                for (i = 0; i < priv->child_count; i++)
-                        if (child_up[i] == 0)
-                                down_children++;
+    local = frame->local;
 
-                if (down_children == priv->child_count) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "All subvolumes are down. Going offline "
-                                "until atleast one of them comes back up.");
+    if (local->op != GF_FOP_LOOKUP)
+        /* TODO:If the replica count is being increased on a plain distribute
+         * volume that was never mounted, we need to allow setxattr on '/' with
+         * GF_CLIENT_PID_NO_ROOT_SQUASH to accomodate for DHT layout setting */
+        return _gf_false;
 
-                        default_notify (this, event, data);
-                }
+    if (local->inode == NULL)
+        return _gf_false;
 
-                break;
+    if (!__is_root_gfid(local->inode->gfid))
+        return _gf_false;
 
-        default:
-                default_notify (this, event, data);
-        }
+    return _gf_true;
+}
 
-        return 0;
+gf_boolean_t
+afr_lookup_has_quorum(call_frame_t *frame, const unsigned int up_children_count)
+{
+    if (frame && (up_children_count > 0) &&
+        afr_is_add_replica_mount_lookup_on_root(frame))
+        return _gf_true;
+
+    return _gf_false;
+}
+
+void
+afr_handle_replies_quorum(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = frame->local;
+    afr_private_t *priv = this->private;
+    unsigned char *success_replies = NULL;
+
+    success_replies = alloca0(priv->child_count);
+    afr_fill_success_replies(local, priv, success_replies);
+
+    if (priv->quorum_count && !afr_has_quorum(success_replies, this, NULL)) {
+        local->op_errno = afr_final_errno(local, priv);
+        if (!local->op_errno)
+            local->op_errno = afr_quorum_errno(priv);
+        local->op_ret = -1;
+    }
+}
+
+gf_boolean_t
+afr_ta_dict_contains_pending_xattr(dict_t *dict, afr_private_t *priv, int child)
+{
+    int *pending = NULL;
+    int ret = 0;
+    int i = 0;
+
+    ret = dict_get_ptr(dict, priv->pending_key[child], (void *)&pending);
+    if (ret == 0) {
+        for (i = 0; i < AFR_NUM_CHANGE_LOGS; i++) {
+            /* Not doing a ntoh32(pending) as we just want to check
+             * if it is non-zero or not. */
+            if (pending[i]) {
+                return _gf_true;
+            }
+        }
+    }
 
+    return _gf_false;
 }
diff --git a/xlators/cluster/afr/src/afr-dir-read.c b/xlators/cluster/afr/src/afr-dir-read.c
index 19ddcbda762..f8bf8340dab 100644
--- a/xlators/cluster/afr/src/afr-dir-read.c
+++ b/xlators/cluster/afr/src/afr-dir-read.c
@@ -1,753 +1,346 @@
 /*
-   Copyright (c) 2007-2010 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
 
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
 
 #include <libgen.h>
 #include <unistd.h>
-#include <fnmatch.h>
 #include <sys/time.h>
 #include <stdlib.h>
 #include <signal.h>
 #include <string.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "stack.h"
-#include "list.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include "compat-errno.h"
-#include "compat.h"
-#include "checksum.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/dict.h>
+#include <glusterfs/list.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/compat.h>
 
 #include "afr.h"
-#include "afr-self-heal.h"
-#include "afr-self-heal-common.h"
-
-
-int
-afr_examine_dir_sh_unwind (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local  = NULL;
-
-        local = frame->local;
-
-        afr_set_opendir_done (this, local->fd->inode);
-
-        AFR_STACK_UNWIND (opendir, frame, local->op_ret,
-                          local->op_errno, local->fd);
-
-        return 0;
-}
-
-
-gf_boolean_t
-__checksums_differ (uint32_t *checksum, int child_count,
-                    unsigned char *child_up)
-{
-        int ret = _gf_false;
-        int i = 0;
-
-        uint32_t cksum;
-
-        cksum = checksum[0];
-
-        for (i = 0; i < child_count; i++) {
-                if (!child_up[i])
-                        continue;
-
-                if (cksum != checksum[i]) {
-                        ret = _gf_true;
-                        break;
-                }
-
-                cksum = checksum[i];
-        }
-
-        return ret;
-}
-
+#include "afr-transaction.h"
 
 int32_t
-afr_examine_dir_readdir_cbk (call_frame_t *frame, void *cookie,
-                             xlator_t *this, int32_t op_ret, int32_t op_errno,
-                             gf_dirent_t *entries)
+afr_opendir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-	afr_private_t *    priv  = NULL;
-	afr_local_t *      local = NULL;
-        afr_self_heal_t *  sh    = NULL;
-
-        gf_dirent_t * entry = NULL;
-        gf_dirent_t * tmp   = NULL;
-
-        int child_index = 0;
-
-        uint32_t entry_cksum;
-
-        int   call_count    = 0;
-        off_t last_offset = 0;
-        char  sh_type_str[256] = {0,};
+    afr_local_t *local = NULL;
+    int call_count = -1;
+    int32_t child_index = 0;
+    afr_fd_ctx_t *fd_ctx = NULL;
 
-        priv  = this->private;
-        local = frame->local;
-        sh    = &local->self_heal;
+    local = frame->local;
+    fd_ctx = local->fd_ctx;
+    child_index = (long)cookie;
 
-        child_index = (long) cookie;
+    local->replies[child_index].valid = 1;
+    local->replies[child_index].op_ret = op_ret;
+    local->replies[child_index].op_errno = op_errno;
 
+    LOCK(&frame->lock);
+    {
         if (op_ret == -1) {
-                local->op_ret = -1;
-                local->op_ret = op_errno;
-                goto out;
+            local->op_errno = op_errno;
+            fd_ctx->opened_on[child_index] = AFR_FD_NOT_OPENED;
+        } else {
+            local->op_ret = op_ret;
+            fd_ctx->opened_on[child_index] = AFR_FD_OPENED;
+            if (!local->xdata_rsp && xdata)
+                local->xdata_rsp = dict_ref(xdata);
         }
+        call_count = --local->call_count;
+    }
+    UNLOCK(&frame->lock);
 
-        if (op_ret == 0)
-                goto out;
+    if (call_count == 0) {
+        afr_handle_replies_quorum(frame, this);
+        AFR_STACK_UNWIND(opendir, frame, local->op_ret, local->op_errno,
+                         local->fd, NULL);
+    }
 
-        list_for_each_entry_safe (entry, tmp, &entries->list, list) {
-                entry_cksum = gf_rsync_weak_checksum (entry->d_name,
-                                                      strlen (entry->d_name));
-                local->cont.opendir.checksum[child_index] ^= entry_cksum;
-        }
-
-        list_for_each_entry (entry, &entries->list, list) {
-                last_offset = entry->d_off;
-        }
-
-        /* read more entries */
-
-        STACK_WIND_COOKIE (frame, afr_examine_dir_readdir_cbk,
-                           (void *) (long) child_index,
-                           priv->children[child_index],
-                           priv->children[child_index]->fops->readdir,
-                           local->fd, 131072, last_offset);
-
-out:
-        if ((op_ret == 0) || (op_ret == -1)) {
-                call_count = afr_frame_return (frame);
-
-                if (call_count == 0) {
-                        if (__checksums_differ (local->cont.opendir.checksum,
-                                                priv->child_count,
-                                                local->child_up)) {
-
-                                sh->need_entry_self_heal  = _gf_true;
-                                sh->forced_merge          = _gf_true;
-                                sh->type                  = local->fd->inode->ia_type;
-                                sh->background            = _gf_false;
-                                sh->unwind                = afr_examine_dir_sh_unwind;
-
-                                afr_self_heal_type_str_get(&local->self_heal,
-                                                           sh_type_str,
-                                                           sizeof(sh_type_str));
-                                gf_log (this->name, GF_LOG_NORMAL,
-                                        "%s self-heal triggered. path: %s, "
-                                         "reason: checksums of directory differ,"
-                                        " forced merge option set",
-                                        sh_type_str, local->loc.path);
-
-                                afr_self_heal (frame, this);
-                        } else {
-                                afr_set_opendir_done (this, local->fd->inode);
-
-                                AFR_STACK_UNWIND (opendir, frame, local->op_ret,
-                                                  local->op_errno, local->fd);
-                        }
-                }
-        }
-
-        return 0;
+    return 0;
 }
 
-
 int
-afr_examine_dir (call_frame_t *frame, xlator_t *this)
-{
-        afr_private_t * priv  = NULL;
-        afr_local_t *   local = NULL;
-
-        int i;
-        int call_count = 0;
-
-        local = frame->local;
-        priv  = this->private;
-
-        local->cont.opendir.checksum = GF_CALLOC (priv->child_count,
-                                        sizeof (*local->cont.opendir.checksum),
-                                        gf_afr_mt_int32_t);
-
-        call_count = afr_up_children_count (priv->child_count, local->child_up);
-
-        local->call_count = call_count;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, afr_examine_dir_readdir_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->readdir,
-                                           local->fd, 131072, 0);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
-
-        return 0;
-}
-
-
-int32_t
-afr_opendir_cbk (call_frame_t *frame, void *cookie,
-		 xlator_t *this, int32_t op_ret, int32_t op_errno,
-		 fd_t *fd)
-{
-        afr_private_t *priv              = NULL;
-	afr_local_t   *local             = NULL;
-        int32_t        up_children_count = 0;
-        int           ret                = -1;
-
-	int call_count = -1;
-
-        priv  = this->private;
-        local = frame->local;
-
-        up_children_count = afr_up_children_count (priv->child_count,
-                                                   local->child_up);
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret >= 0)
-			local->op_ret = op_ret;
-
-		local->op_errno = op_errno;
-	}
-	UNLOCK (&frame->lock);
-
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0) {
-                if (local->op_ret == 0) {
-
-                        ret = afr_fd_ctx_set (this, local->fd);
-
-                        if (ret) {
-                                local->op_ret = -1;
-                                local->op_errno = -1;
-                                gf_log (this->name, GF_LOG_ERROR, " failed to "
-                                        "set fd ctx for fd %p", local->fd);
-                                goto out;
-                        }
-                        if (!afr_is_opendir_done (this, local->fd->inode) &&
-                            up_children_count > 1) {
-
-                                /*
-                                 * This is the first opendir on this inode. We need
-                                 * to check if the directory's entries are the same
-                                 * on all subvolumes. This is needed in addition
-                                 * to regular entry self-heal because the readdir
-                                 * call is sent only to the first subvolume, and
-                                 * thus files that exist only there will never be healed
-                                 * otherwise (assuming changelog shows no anamolies).
-                                 */
-
-                                gf_log (this->name, GF_LOG_TRACE,
-                                        "reading contents of directory %s looking for mismatch",
-                                        local->loc.path);
-
-                                afr_examine_dir (frame, this);
-
-                        } else {
-                                AFR_STACK_UNWIND (opendir, frame, local->op_ret,
-                                                  local->op_errno, local->fd);
-                        }
-                } else {
-out:
-                        AFR_STACK_UNWIND (opendir, frame, local->op_ret,
-                                          local->op_errno, local->fd);
-                }
-        }
-
-	return 0;
-}
-
-
-int32_t 
-afr_opendir (call_frame_t *frame, xlator_t *this,
-	     loc_t *loc, fd_t *fd)
+afr_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+            dict_t *xdata)
 {
-	afr_private_t * priv        = NULL;
-	afr_local_t   * local       = NULL;
-
-	int             child_count = 0;
-	int             i           = 0;
-
-	int ret = -1;
-	int call_count = -1;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int i = 0;
+    int call_count = -1;
+    int32_t op_errno = ENOMEM;
+    afr_fd_ctx_t *fd_ctx = NULL;
 
-	int32_t         op_ret   = -1;
-	int32_t         op_errno = 0;
+    priv = this->private;
 
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-	priv = this->private;
+    local->op = GF_FOP_OPENDIR;
 
-	child_count = priv->child_count;
-
-	ALLOC_OR_GOTO (local, afr_local_t, out);
-	ret = AFR_LOCAL_INIT (local, priv);
-	if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-	}
-
-        loc_copy (&local->loc, loc);
-
-	frame->local = local;
-	local->fd    = fd_ref (fd);
-
-	call_count = local->call_count;
-	
-	for (i = 0; i < child_count; i++) {
-		if (local->child_up[i]) {
-			STACK_WIND (frame, afr_opendir_cbk, 
-				    priv->children[i],
-				    priv->children[i]->fops->opendir,
-				    loc, fd);
-
-			if (!--call_count)
-				break;
-		}
-	}
-
-	op_ret = 0;
-out:
-	if (op_ret == -1) {
-		AFR_STACK_UNWIND (opendir, frame, op_ret, op_errno, fd);
-	}
+    if (priv->quorum_count && !afr_has_quorum(local->child_up, this, NULL)) {
+        op_errno = afr_quorum_errno(priv);
+        goto out;
+    }
 
-	return 0;
-}
+    if (!afr_is_consistent_io_possible(local, priv, &op_errno))
+        goto out;
 
+    fd_ctx = afr_fd_ctx_get(fd, this);
+    if (!fd_ctx)
+        goto out;
 
-/**
- * Common algorithm for directory read calls:
- * 
- * - Try the fop on the first child that is up
- * - if we have failed due to ENOTCONN:
- *     try the next child
- *
- * Applicable to: readdir
- */
+    loc_copy(&local->loc, loc);
 
+    local->fd = fd_ref(fd);
+    local->fd_ctx = fd_ctx;
 
-struct entry_name {
-        char *name;
-        struct list_head list;
-};
+    call_count = local->call_count;
 
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->child_up[i]) {
+            STACK_WIND_COOKIE(frame, afr_opendir_cbk, (void *)(long)i,
+                              priv->children[i],
+                              priv->children[i]->fops->opendir, loc, fd, NULL);
 
-static gf_boolean_t
-remembered_name (const char *name, struct list_head *entries)
-{
-        struct entry_name *e;
-        gf_boolean_t ret = _gf_false;
-
-        list_for_each_entry (e, entries, list) {
-                if (!strcmp (name, e->name)) {
-                        ret = _gf_true;
-                        goto out;
-                }
+            if (!--call_count)
+                break;
         }
+    }
 
+    return 0;
 out:
-        return ret;
+    AFR_STACK_UNWIND(opendir, frame, -1, op_errno, fd, NULL);
+    return 0;
 }
 
-
-static void
-afr_remember_entries (gf_dirent_t *entries, fd_t *fd)
+static int
+afr_validate_read_subvol(inode_t *inode, xlator_t *this, int par_read_subvol)
 {
-	struct entry_name *n       = NULL;
-	gf_dirent_t *      entry   = NULL;
-
-	int ret = 0;
-
-	uint64_t      ctx;
-	afr_fd_ctx_t *fd_ctx;
-
-	ret = fd_ctx_get (fd, THIS, &ctx);
-	if (ret < 0) {
-		gf_log (THIS->name, GF_LOG_DEBUG,
-			"could not get fd ctx for fd=%p", fd);
-		return;
-	}
-
-        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
-
-	list_for_each_entry (entry, &entries->list, list) {
-		n = GF_CALLOC (1, sizeof (*n), gf_afr_mt_entry_name);
-		n->name = gf_strdup (entry->d_name);
-		INIT_LIST_HEAD (&n->list);
+    int gen = 0;
+    int entry_read_subvol = 0;
+    unsigned char *data_readable = NULL;
+    unsigned char *metadata_readable = NULL;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+    data_readable = alloca0(priv->child_count);
+    metadata_readable = alloca0(priv->child_count);
+
+    afr_inode_read_subvol_get(inode, this, data_readable, metadata_readable,
+                              &gen);
+
+    if (gen != priv->event_generation || !data_readable[par_read_subvol] ||
+        !metadata_readable[par_read_subvol])
+        return -1;
+
+    /* Once the control reaches the following statement, it means that the
+     * parent's read subvol is perfectly readable. So calling
+     * either afr_data_subvol_get() or afr_metadata_subvol_get() would
+     * yield the same result. Hence, choosing afr_data_subvol_get() below.
+     */
+
+    if (!priv->consistent_metadata)
+        return 0;
 
-		list_add (&n->list, &fd_ctx->entries);
-	}
+    /* For an inode fetched through readdirp which is yet to be linked,
+     * inode ctx would not be initialised (yet). So this function returns
+     * -1 above due to gen being 0, which is why it is OK to pass NULL for
+     *  read_subvol_args here.
+     */
+    entry_read_subvol = afr_data_subvol_get(inode, this, NULL, NULL, NULL,
+                                            NULL);
+    if (entry_read_subvol != par_read_subvol)
+        return -1;
+
+    return 0;
 }
 
-
-static off_t
-afr_filter_entries (gf_dirent_t *entries, fd_t *fd)
+static void
+afr_readdir_transform_entries(call_frame_t *frame, gf_dirent_t *subvol_entries,
+                              int subvol, gf_dirent_t *entries, fd_t *fd)
 {
-	gf_dirent_t *entry, *tmp;
-	int ret = 0;
-
-	uint64_t      ctx;
-	afr_fd_ctx_t *fd_ctx;
-
-	off_t offset = 0;
-
-	ret = fd_ctx_get (fd, THIS, &ctx);
-	if (ret < 0) {
-		gf_log (THIS->name, GF_LOG_DEBUG,
-			"could not get fd ctx for fd=%p", fd);
-		return -1;
-	}
-
-        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
-
-	list_for_each_entry_safe (entry, tmp, &entries->list, list) {
-		offset = entry->d_off;
-
-		if (remembered_name (entry->d_name, &fd_ctx->entries)) {
-			list_del (&entry->list);
-			GF_FREE (entry);
-		}
-	}
+    int ret = -1;
+    gf_dirent_t *entry = NULL;
+    gf_dirent_t *tmp = NULL;
+    xlator_t *this = NULL;
+    afr_private_t *priv = NULL;
+    gf_boolean_t need_heal = _gf_false;
+    gf_boolean_t validate_subvol = _gf_false;
+
+    this = THIS;
+    priv = this->private;
+
+    need_heal = afr_get_need_heal(this);
+    validate_subvol = need_heal | priv->consistent_metadata;
+
+    list_for_each_entry_safe(entry, tmp, &subvol_entries->list, list)
+    {
+        if (afr_is_private_directory(priv, fd->inode->gfid, entry->d_name,
+                                     frame->root->pid)) {
+            continue;
+        }
 
-	return offset;
-}
+        list_del_init(&entry->list);
+        list_add_tail(&entry->list, &entries->list);
 
+        if (!validate_subvol)
+            continue;
 
-static void
-afr_forget_entries (fd_t *fd)
-{
-	struct entry_name *entry, *tmp;
-	int ret = 0;
-
-	uint64_t      ctx;
-	afr_fd_ctx_t *fd_ctx;
-
-	ret = fd_ctx_get (fd, THIS, &ctx);
-	if (ret < 0) {
-		gf_log (THIS->name, GF_LOG_DEBUG,
-			"could not get fd ctx for fd=%p", fd);
-		return;
-	}
-
-        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
-
-	list_for_each_entry_safe (entry, tmp, &fd_ctx->entries, list) {
-		GF_FREE (entry->name);
-		list_del (&entry->list);
-		GF_FREE (entry);
-	}
+        if (entry->inode) {
+            ret = afr_validate_read_subvol(entry->inode, this, subvol);
+            if (ret == -1) {
+                inode_unref(entry->inode);
+                entry->inode = NULL;
+                continue;
+            }
+        }
+    }
 }
 
-
 int32_t
-afr_readdir_cbk (call_frame_t *frame, void *cookie,
-		 xlator_t *this, int32_t op_ret, int32_t op_errno,
-		 gf_dirent_t *entries)
+afr_readdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, gf_dirent_t *subvol_entries,
+                dict_t *xdata)
 {
-	afr_private_t * priv     = NULL;
-	afr_local_t *   local    = NULL;
+    afr_local_t *local = NULL;
+    gf_dirent_t entries;
 
-        gf_dirent_t * entry = NULL;
-        gf_dirent_t * tmp   = NULL;
+    INIT_LIST_HEAD(&entries.list);
 
-        int child_index = -1;
+    local = frame->local;
 
-	priv     = this->private;
-	local = frame->local;
-        child_index = (long) cookie;
+    if (op_ret < 0 && !local->cont.readdir.offset) {
+        /* failover only if this was first readdir, detected
+           by offset == 0 */
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
 
-	if (op_ret != -1) {
-                list_for_each_entry_safe (entry, tmp, &entries->list, list) {
-                        entry->d_ino = afr_itransform (entry->d_ino,
-                                                       priv->child_count,
-                                                       child_index);
+        afr_read_txn_continue(frame, this, (long)cookie);
+        return 0;
+    }
 
-                        if ((local->fd->inode == local->fd->inode->table->root)
-                            && !strcmp (entry->d_name, GF_REPLICATE_TRASH_DIR)) {
-                                list_del_init (&entry->list);
-                                GF_FREE (entry);
-                        }
-                }
-    	}
+    if (op_ret >= 0)
+        afr_readdir_transform_entries(frame, subvol_entries, (long)cookie,
+                                      &entries, local->fd);
 
-        AFR_STACK_UNWIND (readdir, frame, op_ret, op_errno, entries);
+    AFR_STACK_UNWIND(readdir, frame, op_ret, op_errno, &entries, xdata);
 
-	return 0;
-}
+    gf_dirent_free(&entries);
 
+    return 0;
+}
 
-int32_t
-afr_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, gf_dirent_t *entries)
+int
+afr_readdir_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_private_t * priv     = NULL;
-        afr_local_t *   local    = NULL;
-        xlator_t **     children = NULL;
-        ino_t           inum = 0;
-
-        int call_child = 0;
-        int ret        = 0;
-
-        gf_dirent_t * entry = NULL;
-        gf_dirent_t * tmp   = NULL;
-
-        int child_index = -1;
-
-        uint64_t      ctx    = 0;
-        afr_fd_ctx_t *fd_ctx = NULL;
-
-	off_t offset = 0;
-
-        priv     = this->private;
-        children = priv->children;
-
-        local = frame->local;
-
-        child_index = (long) cookie;
-
-	if (priv->strict_readdir) {
-		ret = fd_ctx_get (local->fd, this, &ctx);
-		if (ret < 0) {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"could not get fd ctx for fd=%p", local->fd);
-			op_ret   = -1;
-			op_errno = -ret;
-			goto out;
-		}
-
-		fd_ctx = (afr_fd_ctx_t *)(long) ctx;
-
-		if (child_went_down (op_ret, op_errno)) {
-			if (all_tried (child_index, priv->child_count)) {
-				goto out;
-			}
-
-			call_child = ++child_index;
-
-			gf_log (this->name, GF_LOG_TRACE,
-				"starting readdir afresh on child %d, offset %"PRId64,
-				call_child, (uint64_t) 0);
-
-			fd_ctx->failed_over = _gf_true;
-
-			STACK_WIND_COOKIE (frame, afr_readdirp_cbk,
-					   (void *) (long) call_child,
-					   children[call_child],
-					   children[call_child]->fops->readdirp, local->fd,
-					   local->cont.readdir.size, 0);
-			return 0;
-		}
-	}
-
-	if (op_ret != -1) {
-		list_for_each_entry_safe (entry, tmp, &entries->list, list) {
-			inum = afr_itransform (entry->d_ino, priv->child_count,
-					       child_index);
-			entry->d_ino = inum;
-			inum  = afr_itransform (entry->d_stat.ia_ino,
-						priv->child_count, child_index);
-			entry->d_stat.ia_ino = inum;
-
-			if ((local->fd->inode == local->fd->inode->table->root)
-			    && !strcmp (entry->d_name, GF_REPLICATE_TRASH_DIR)) {
-				list_del_init (&entry->list);
-				GF_FREE (entry);
-			}
-		}
-	}
-
-	if (priv->strict_readdir) {
-		if (fd_ctx->failed_over) {
-			if (list_empty (&entries->list)) {
-				goto out;
-			}
-
-			offset = afr_filter_entries (entries, local->fd);
-
-			afr_remember_entries (entries, local->fd);
-
-			if (list_empty (&entries->list)) {
-				/* All the entries we got were duplicate. We
-				   shouldn't send an empty list now, because
-				   that'll make the application stop reading. So
-				   try to get more entries */
-
-				gf_log (this->name, GF_LOG_TRACE,
-					"trying to fetch non-duplicate entries from offset %"PRId64", child %s",
-					offset, children[child_index]->name);
-
-				STACK_WIND_COOKIE (frame, afr_readdirp_cbk,
-						   (void *) (long) child_index,
-						   children[child_index],
-						   children[child_index]->fops->readdirp,
-						   local->fd, local->cont.readdir.size, offset);
-				return 0;
-			}
-		} else {
-			afr_remember_entries (entries, local->fd);
-		}
-	}
-
-out:
-        AFR_STACK_UNWIND (readdirp, frame, op_ret, op_errno, entries);
-
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    afr_fd_ctx_t *fd_ctx = NULL;
+
+    priv = this->private;
+    local = frame->local;
+    fd_ctx = afr_fd_ctx_get(local->fd, this);
+    if (!fd_ctx) {
+        local->op_errno = EINVAL;
+        local->op_ret = -1;
+    }
+
+    if (subvol == -1 || !fd_ctx) {
+        AFR_STACK_UNWIND(readdir, frame, local->op_ret, local->op_errno, 0, 0);
         return 0;
+    }
+
+    fd_ctx->readdir_subvol = subvol;
+
+    if (local->op == GF_FOP_READDIR)
+        STACK_WIND_COOKIE(frame, afr_readdir_cbk, (void *)(long)subvol,
+                          priv->children[subvol],
+                          priv->children[subvol]->fops->readdir, local->fd,
+                          local->cont.readdir.size, local->cont.readdir.offset,
+                          local->xdata_req);
+    else
+        STACK_WIND_COOKIE(frame, afr_readdir_cbk, (void *)(long)subvol,
+                          priv->children[subvol],
+                          priv->children[subvol]->fops->readdirp, local->fd,
+                          local->cont.readdir.size, local->cont.readdir.offset,
+                          local->xdata_req);
+    return 0;
 }
 
-
-int32_t
-afr_do_readdir (call_frame_t *frame, xlator_t *this,
-	        fd_t *fd, size_t size, off_t offset, int whichop)
+int
+afr_do_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+               off_t offset, int whichop, dict_t *dict)
 {
-	afr_private_t * priv       = NULL;
-	xlator_t **     children   = NULL;
-	int             call_child = 0;
-	afr_local_t     *local     = NULL;
-
-        uint64_t      ctx;
-        afr_fd_ctx_t *fd_ctx;
-
-	int ret = -1;
-
-	int32_t op_ret   = -1;
-	int32_t op_errno = 0;
-
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
-
-	priv     = this->private;
-	children = priv->children;
-
-	ALLOC_OR_GOTO (local, afr_local_t, out);
-	ret = AFR_LOCAL_INIT (local, priv);
-	if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-	}
-
-	frame->local = local;
-
-	call_child = afr_first_up_child (priv);
-	if (call_child == -1) {
-		op_errno = ENOTCONN;
-		gf_log (this->name, GF_LOG_DEBUG,
-			"no child is up");
-		goto out;
-	}
-
-        local->fd                  = fd_ref (fd);
-        local->cont.readdir.size   = size;
-
-	if (priv->strict_readdir) {
-		ret = fd_ctx_get (fd, this, &ctx);
-		if (ret < 0) {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"could not get fd ctx for fd=%p", fd);
-			op_errno = -ret;
-			goto out;
-		}
-
-		fd_ctx = (afr_fd_ctx_t *)(long) ctx;
-
-		if (fd_ctx->last_tried != call_child) {
-			gf_log (this->name, GF_LOG_TRACE,
-				"first up child has changed from %d to %d, restarting readdir from offset 0",
-				fd_ctx->last_tried, call_child);
-
-			fd_ctx->failed_over = _gf_true;
-			offset = 0;
-		}
-
-		fd_ctx->last_tried = call_child;
-	}
-
-        if (whichop == GF_FOP_READDIR)
-                STACK_WIND_COOKIE (frame, afr_readdir_cbk,
-                                   (void *) (long) call_child,
-                                   children[call_child],
-                                   children[call_child]->fops->readdir, fd,
-                                   size, offset);
-        else
-                STACK_WIND_COOKIE (frame, afr_readdirp_cbk,
-                                   (void *) (long) call_child,
-                                   children[call_child],
-                                   children[call_child]->fops->readdirp, fd,
-                                   size, offset);
-
-	op_ret = 0;
+    afr_local_t *local = NULL;
+    int32_t op_errno = 0;
+    int subvol = -1;
+    afr_fd_ctx_t *fd_ctx = NULL;
+
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
+
+    fd_ctx = afr_fd_ctx_get(fd, this);
+    if (!fd_ctx) {
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    local->op = whichop;
+    local->fd = fd_ref(fd);
+    local->cont.readdir.size = size;
+    local->cont.readdir.offset = offset;
+    local->xdata_req = (dict) ? dict_ref(dict) : NULL;
+
+    subvol = fd_ctx->readdir_subvol;
+
+    if (offset == 0 || subvol == -1) {
+        /* First readdir has option of failing over and selecting
+           an appropriate read subvolume */
+        afr_read_txn(frame, this, fd->inode, afr_readdir_wind,
+                     AFR_DATA_TRANSACTION);
+    } else {
+        /* But continued readdirs MUST stick to the same subvolume
+           without an option to failover */
+        afr_readdir_wind(frame, this, subvol);
+    }
+
+    return 0;
 out:
-	if (op_ret == -1) {
-		AFR_STACK_UNWIND (readdir, frame, op_ret, op_errno, NULL);
-	}
-	return 0;
+    AFR_STACK_UNWIND(readdir, frame, -1, op_errno, NULL, NULL);
+    return 0;
 }
 
-
 int32_t
-afr_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-             off_t offset)
+afr_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+            off_t offset, dict_t *xdata)
 {
-        afr_do_readdir (frame, this, fd, size, offset, GF_FOP_READDIR);
-        return 0;
-}
+    afr_do_readdir(frame, this, fd, size, offset, GF_FOP_READDIR, xdata);
 
+    return 0;
+}
 
 int32_t
-afr_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-              off_t offset)
+afr_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+             off_t offset, dict_t *dict)
 {
-        afr_do_readdir (frame, this, fd, size, offset, GF_FOP_READDIRP);
-        return 0;
-}
+    afr_do_readdir(frame, this, fd, size, offset, GF_FOP_READDIRP, dict);
 
+    return 0;
+}
 
 int32_t
-afr_releasedir (xlator_t *this, fd_t *fd)
+afr_releasedir(xlator_t *this, fd_t *fd)
 {
-	afr_forget_entries (fd);
-        afr_cleanup_fd_ctx (this, fd);
+    afr_cleanup_fd_ctx(this, fd);
 
-	return 0;
+    return 0;
 }
diff --git a/xlators/cluster/afr/src/afr-dir-read.h b/xlators/cluster/afr/src/afr-dir-read.h
index 40c7b6aef28..773e925ec6c 100644
--- a/xlators/cluster/afr/src/afr-dir-read.h
+++ b/xlators/cluster/afr/src/afr-dir-read.h
@@ -1,50 +1,33 @@
 /*
-   Copyright (c) 2007-2010 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __DIR_READ_H__
 #define __DIR_READ_H__
 
-
 int32_t
-afr_opendir (call_frame_t *frame, xlator_t *this,
-	     loc_t *loc, fd_t *fd);
+afr_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+            dict_t *xdata);
 
 int32_t
-afr_releasedir (xlator_t *this, fd_t *fd);
+afr_releasedir(xlator_t *this, fd_t *fd);
 
 int32_t
-afr_readdir (call_frame_t *frame, xlator_t *this,
-	     fd_t *fd, size_t size, off_t offset);
-
+afr_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+            off_t offset, dict_t *xdata);
 
 int32_t
-afr_readdirp (call_frame_t *frame, xlator_t *this,
-	     fd_t *fd, size_t size, off_t offset);
+afr_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+             off_t offset, dict_t *dict);
 
 int32_t
-afr_getdents (call_frame_t *frame, xlator_t *this,
-	      fd_t *fd, size_t size, off_t offset, int32_t flag);
-
-
-int32_t
-afr_checksum (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, int32_t flags);
-
+afr_checksum(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+             dict_t *xdata);
 
 #endif /* __DIR_READ_H__ */
diff --git a/xlators/cluster/afr/src/afr-dir-write.c b/xlators/cluster/afr/src/afr-dir-write.c
index af42e7e06a0..b7cceb79158 100644
--- a/xlators/cluster/afr/src/afr-dir-write.c
+++ b/xlators/cluster/afr/src/afr-dir-write.c
@@ -1,1349 +1,937 @@
 /*
-  Copyright (c) 2007-2010 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
-
 #include <libgen.h>
 #include <unistd.h>
-#include <fnmatch.h>
 #include <sys/time.h>
 #include <stdlib.h>
 #include <signal.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
+#include <glusterfs/glusterfs.h>
 #include "afr.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "stack.h"
-#include "list.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include "compat-errno.h"
-#include "compat.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/list.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/byte-order.h>
 
 #include "afr.h"
 #include "afr-transaction.h"
 
-
 void
-afr_build_parent_loc (loc_t *parent, loc_t *child)
+afr_mark_entry_pending_changelog(call_frame_t *frame, xlator_t *this);
+
+int
+afr_build_parent_loc(loc_t *parent, loc_t *child, int32_t *op_errno)
 {
-	char *tmp = NULL;
+    int ret = -1;
+    char *child_path = NULL;
+
+    if (!child->parent) {
+        if (op_errno)
+            *op_errno = EINVAL;
+        goto out;
+    }
+
+    child_path = gf_strdup(child->path);
+    if (!child_path) {
+        if (op_errno)
+            *op_errno = ENOMEM;
+        goto out;
+    }
+
+    parent->path = gf_strdup(dirname(child_path));
+    if (!parent->path) {
+        if (op_errno)
+            *op_errno = ENOMEM;
+        goto out;
+    }
+
+    parent->inode = inode_ref(child->parent);
+    gf_uuid_copy(parent->gfid, child->pargfid);
+
+    ret = 0;
+out:
+    GF_FREE(child_path);
 
-	if (!child->parent) {
-		loc_copy (parent, child);
-		return;
-	}
+    return ret;
+}
 
-	tmp = gf_strdup (child->path);
-	parent->path   = gf_strdup (dirname (tmp));
-	GF_FREE (tmp);
+static void
+__afr_dir_write_finalize(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int inode_read_subvol = -1;
+    int parent_read_subvol = -1;
+    int parent2_read_subvol = -1;
+    int i = 0;
+    afr_read_subvol_args_t args = {
+        0,
+    };
+
+    local = frame->local;
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid)
+            continue;
+        if (local->replies[i].op_ret == -1)
+            continue;
+        gf_uuid_copy(args.gfid, local->replies[i].poststat.ia_gfid);
+        args.ia_type = local->replies[i].poststat.ia_type;
+        break;
+    }
+
+    if (local->inode) {
+        if (local->op != GF_FOP_RENAME && local->op != GF_FOP_LINK)
+            afr_replies_interpret(frame, this, local->inode, NULL);
+
+        inode_read_subvol = afr_data_subvol_get(local->inode, this, NULL, NULL,
+                                                NULL, &args);
+    }
+
+    if (local->parent)
+        parent_read_subvol = afr_data_subvol_get(local->parent, this, NULL,
+                                                 local->readable, NULL, NULL);
+
+    if (local->parent2)
+        parent2_read_subvol = afr_data_subvol_get(local->parent2, this, NULL,
+                                                  local->readable2, NULL, NULL);
+
+    local->op_ret = -1;
+    local->op_errno = afr_final_errno(local, priv);
+    afr_pick_error_xdata(local, priv, local->parent, local->readable,
+                         local->parent2, local->readable2);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid)
+            continue;
+        if (local->replies[i].op_ret < 0) {
+            if (local->inode)
+                afr_inode_need_refresh_set(local->inode, this);
+            if (local->parent)
+                afr_inode_need_refresh_set(local->parent, this);
+            if (local->parent2)
+                afr_inode_need_refresh_set(local->parent2, this);
+            continue;
+        }
 
-        parent->name   = strrchr (parent->path, '/');
-	if (parent->name)
-		parent->name++;
+        if (local->op_ret == -1) {
+            local->op_ret = local->replies[i].op_ret;
+            local->op_errno = local->replies[i].op_errno;
+
+            local->cont.dir_fop.buf = local->replies[i].poststat;
+            local->cont.dir_fop.preparent = local->replies[i].preparent;
+            local->cont.dir_fop.postparent = local->replies[i].postparent;
+            local->cont.dir_fop.prenewparent = local->replies[i].preparent2;
+            local->cont.dir_fop.postnewparent = local->replies[i].postparent2;
+            if (local->xdata_rsp) {
+                dict_unref(local->xdata_rsp);
+                local->xdata_rsp = NULL;
+            }
+
+            if (local->replies[i].xdata)
+                local->xdata_rsp = dict_ref(local->replies[i].xdata);
+            continue;
+        }
 
-	parent->inode  = inode_ref (child->parent);
-	parent->parent = inode_parent (parent->inode, 0, NULL);
-	parent->ino    = parent->inode->ino;
-}
+        if (i == inode_read_subvol) {
+            local->cont.dir_fop.buf = local->replies[i].poststat;
+            if (local->replies[i].xdata) {
+                if (local->xdata_rsp)
+                    dict_unref(local->xdata_rsp);
+                local->xdata_rsp = dict_ref(local->replies[i].xdata);
+            }
+        }
 
-/* {{{ create */
+        if (i == parent_read_subvol) {
+            local->cont.dir_fop.preparent = local->replies[i].preparent;
+            local->cont.dir_fop.postparent = local->replies[i].postparent;
+        }
 
-int
-afr_create_unwind (call_frame_t *frame, xlator_t *this)
-{
-	call_frame_t *main_frame = NULL;
-	afr_local_t  *local = NULL;
-        struct iatt  *unwind_buf = NULL;
-
-	local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		if (local->transaction.main_frame) {
-			main_frame = local->transaction.main_frame;
-		}
-		local->transaction.main_frame = NULL;
-	}
-	UNLOCK (&frame->lock);
-
-	if (main_frame) {
-                if (local->cont.create.read_child_buf.ia_ino) {
-                        unwind_buf = &local->cont.create.read_child_buf;
-                } else {
-                        unwind_buf = &local->cont.create.buf;
-                }
-
-                unwind_buf->ia_ino = local->cont.create.ino;
-
-                local->cont.create.preparent.ia_ino  = local->cont.create.parent_ino;
-                local->cont.create.postparent.ia_ino = local->cont.create.parent_ino;
-
-		AFR_STACK_UNWIND (create, main_frame,
-                                  local->op_ret, local->op_errno,
-				  local->cont.create.fd,
-				  local->cont.create.inode,
-				  unwind_buf, &local->cont.create.preparent,
-                                  &local->cont.create.postparent);
+        if (i == parent2_read_subvol) {
+            local->cont.dir_fop.prenewparent = local->replies[i].preparent2;
+            local->cont.dir_fop.postnewparent = local->replies[i].postparent2;
         }
-        
-	return 0;
+    }
 }
 
-
-int
-afr_create_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this, 
-		     int32_t op_ret, int32_t op_errno, 
-		     fd_t *fd, inode_t *inode, struct iatt *buf,
-                     struct iatt *preparent, struct iatt *postparent)
+static void
+__afr_dir_write_fill(call_frame_t *frame, xlator_t *this, int child_index,
+                     int op_ret, int op_errno, struct iatt *poststat,
+                     struct iatt *preparent, struct iatt *postparent,
+                     struct iatt *preparent2, struct iatt *postparent2,
+                     dict_t *xdata)
 {
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
-
-        uint64_t      ctx;
-        afr_fd_ctx_t *fd_ctx;
-
-        int ret = 0;
-
-	int call_count = -1;
-	int child_index = -1;
-
-	local = frame->local;
-	priv  = this->private;
-
-	child_index = (long) cookie;
-
-	LOCK (&frame->lock);
-	{
-		if (afr_fop_failed (op_ret, op_errno))
-			afr_transaction_fop_failed (frame, this, child_index);
-                
-		if (op_ret != -1) {
-			local->op_ret = op_ret;
-
-                        ret = afr_fd_ctx_set (this, fd);
-
-                        if (ret < 0) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "could not set ctx on fd=%p", fd);
-
-                                local->op_ret   = -1;
-                                local->op_errno = -ret;
-                                goto unlock;
-                        }
-
-                        ret = fd_ctx_get (fd, this, &ctx);
-
-                        if (ret < 0) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "could not get fd ctx for fd=%p", fd);
-                                local->op_ret   = -1;
-                                local->op_errno = -ret;
-                                goto unlock;
-                        }
-
-                        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
-
-                        fd_ctx->opened_on[child_index] = 1;
-                        fd_ctx->flags                  = local->cont.create.flags;
-
-                        if (local->success_count == 0) {
-				local->cont.create.buf        = *buf;
-
-                                local->cont.create.ino =
-                                        afr_itransform (buf->ia_ino,
-                                                        priv->child_count,
-                                                        child_index);
-
-                                if (priv->read_child >= 0) {
-                                        afr_set_read_child (this, inode, 
-                                                            priv->read_child);
-                                } else {
-                                        afr_set_read_child (this, inode, 
-                                                            local->read_child_index);
-                                }
-			}
-                        
-                        if (child_index == local->first_up_child) {
-                                local->cont.create.ino =
-                                        afr_itransform (buf->ia_ino,
-                                                        priv->child_count,
-                                                        local->first_up_child);
-                        }
-                        
-                        if (child_index == local->read_child_index) {
-                                local->cont.create.read_child_buf = *buf;
-                                local->cont.create.preparent      = *preparent;
-                                local->cont.create.postparent     = *postparent;
-                        }
-
-			local->cont.create.inode = inode;
-
-			local->success_count++;
-		}
-
-		local->op_errno = op_errno;
-	}
-
-unlock:
-	UNLOCK (&frame->lock);
-
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0) {
-		local->transaction.unwind (frame, this);
-
-		local->transaction.resume (frame, this);
-	}
-	
-	return 0;
+    afr_local_t *local = NULL;
+    afr_fd_ctx_t *fd_ctx = NULL;
+
+    local = frame->local;
+    fd_ctx = local->fd_ctx;
+
+    local->replies[child_index].valid = 1;
+    local->replies[child_index].op_ret = op_ret;
+    local->replies[child_index].op_errno = op_errno;
+    if (xdata)
+        local->replies[child_index].xdata = dict_ref(xdata);
+
+    if (op_ret >= 0) {
+        if (poststat)
+            local->replies[child_index].poststat = *poststat;
+        if (preparent)
+            local->replies[child_index].preparent = *preparent;
+        if (postparent)
+            local->replies[child_index].postparent = *postparent;
+        if (preparent2)
+            local->replies[child_index].preparent2 = *preparent2;
+        if (postparent2)
+            local->replies[child_index].postparent2 = *postparent2;
+        if (fd_ctx)
+            fd_ctx->opened_on[child_index] = AFR_FD_OPENED;
+    } else {
+        if (op_errno != ENOTEMPTY)
+            afr_transaction_fop_failed(frame, this, child_index);
+        if (fd_ctx)
+            fd_ctx->opened_on[child_index] = AFR_FD_NOT_OPENED;
+    }
+
+    return;
 }
 
-
-int
-afr_create_wind (call_frame_t *frame, xlator_t *this)
+static int
+__afr_dir_write_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int op_ret, int op_errno, struct iatt *buf,
+                    struct iatt *preparent, struct iatt *postparent,
+                    struct iatt *preparent2, struct iatt *postparent2,
+                    dict_t *xdata)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-
-	int call_count = -1;
-	int i = 0;
-
-	local = frame->local;
-	priv = this->private;
-
-	call_count = afr_up_children_count (priv->child_count, local->child_up);
-
-	if (call_count == 0) {
-		local->transaction.resume (frame, this);
-		return 0;
-	}
-
-	local->call_count = call_count;
-
-	for (i = 0; i < priv->child_count; i++) {				
-		if (local->child_up[i]) {
-			STACK_WIND_COOKIE (frame, afr_create_wind_cbk,
-					   (void *) (long) i,
-					   priv->children[i], 
-					   priv->children[i]->fops->create,
-					   &local->loc, 
-					   local->cont.create.flags, 
-					   local->cont.create.mode, 
-					   local->cont.create.fd,
-                                           local->cont.create.params);
-			if (!--call_count)
-				break;
-		}
-	}
-	
-	return 0;
-}
+    afr_local_t *local = NULL;
+    int child_index = (long)cookie;
+    int call_count = -1;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+    local = frame->local;
+
+    LOCK(&frame->lock);
+    {
+        __afr_dir_write_fill(frame, this, child_index, op_ret, op_errno, buf,
+                             preparent, postparent, preparent2, postparent2,
+                             xdata);
+        call_count = --local->call_count;
+    }
+    UNLOCK(&frame->lock);
+
+    if (call_count == 0) {
+        __afr_dir_write_finalize(frame, this);
+
+        if (afr_txn_nothing_failed(frame, this)) {
+            /*if it did pre-op, it will do post-op changing ctime*/
+            if (priv->consistent_metadata && afr_needs_changelog_update(local))
+                afr_zero_fill_stat(local);
+            local->transaction.unwind(frame, this);
+        }
+
+        afr_mark_entry_pending_changelog(frame, this);
 
+        afr_transaction_resume(frame, this);
+    }
+
+    return 0;
+}
 
 int
-afr_create_done (call_frame_t *frame, xlator_t *this)
+afr_mark_new_entry_changelog_cbk(call_frame_t *frame, void *cookie,
+                                 xlator_t *this, int op_ret, int op_errno,
+                                 dict_t *xattr, dict_t *xdata)
 {
-	afr_local_t * local = NULL;
+    int call_count = 0;
 
-	local = frame->local;
+    call_count = afr_frame_return(frame);
 
-	local->transaction.unwind (frame, this);
+    if (call_count == 0)
+        AFR_STACK_DESTROY(frame);
 
-	AFR_STACK_DESTROY (frame);
-
-	return 0;
+    return 0;
 }
 
-
-int
-afr_create (call_frame_t *frame, xlator_t *this,
-	    loc_t *loc, int32_t flags, mode_t mode,
-            fd_t *fd, dict_t *params)
+void
+afr_mark_new_entry_changelog(call_frame_t *frame, xlator_t *this)
 {
-	afr_private_t * priv  = NULL;
-	afr_local_t   * local = NULL;
-	call_frame_t  * transaction_frame = NULL;
-
-	int ret = -1;
+    call_frame_t *new_frame = NULL;
+    afr_local_t *local = NULL;
+    afr_local_t *new_local = NULL;
+    afr_private_t *priv = NULL;
+    dict_t *xattr = NULL;
+    int32_t **changelog = NULL;
+    int i = 0;
+    int op_errno = ENOMEM;
+    unsigned char *pending = NULL;
+    int call_count = 0;
+
+    local = frame->local;
+    priv = this->private;
+
+    new_frame = copy_frame(frame);
+    if (!new_frame)
+        goto out;
+
+    new_local = AFR_FRAME_INIT(new_frame, op_errno);
+    if (!new_local)
+        goto out;
+
+    xattr = dict_new();
+    if (!xattr)
+        goto out;
+
+    pending = alloca0(priv->child_count);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->transaction.pre_op[i] &&
+            !local->transaction.failed_subvols[i]) {
+            call_count++;
+            continue;
+        }
+        pending[i] = 1;
+    }
 
-	int op_ret   = -1;
-	int op_errno = 0;
+    changelog = afr_mark_pending_changelog(priv, pending, xattr,
+                                           local->cont.dir_fop.buf.ia_type);
+    if (!changelog)
+        goto out;
 
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
+    new_local->pending = changelog;
+    gf_uuid_copy(new_local->loc.gfid, local->cont.dir_fop.buf.ia_gfid);
+    new_local->loc.inode = inode_ref(local->inode);
 
-	priv = this->private;
+    new_local->call_count = call_count;
 
-	transaction_frame = copy_frame (frame);
-	if (!transaction_frame) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory.");
-		goto out;
-	}
+    for (i = 0; i < priv->child_count; i++) {
+        if (pending[i])
+            continue;
 
-	ALLOC_OR_GOTO (local, afr_local_t, out);
+        STACK_WIND_COOKIE(new_frame, afr_mark_new_entry_changelog_cbk,
+                          (void *)(long)i, priv->children[i],
+                          priv->children[i]->fops->xattrop, &new_local->loc,
+                          GF_XATTROP_ADD_ARRAY, xattr, NULL);
+        if (!--call_count)
+            break;
+    }
 
-	ret = AFR_LOCAL_INIT (local, priv);
-	if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-	}
+    new_frame = NULL;
+out:
+    if (new_frame)
+        AFR_STACK_DESTROY(new_frame);
+    if (xattr)
+        dict_unref(xattr);
+    return;
+}
 
-	transaction_frame->local = local;
+void
+afr_mark_entry_pending_changelog(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int pre_op_count = 0;
+    int failed_count = 0;
+    unsigned char *success_replies = NULL;
 
-	loc_copy (&local->loc, loc);
+    local = frame->local;
+    priv = this->private;
 
-        LOCK (&priv->read_child_lock);
-        {
-                local->read_child_index = (++priv->read_child_rr) 
-                        % (priv->child_count);
-        }
-        UNLOCK (&priv->read_child_lock);
+    if (local->op_ret < 0)
+        return;
 
-	local->cont.create.flags = flags;
-	local->cont.create.mode  = mode;
-	local->cont.create.fd    = fd_ref (fd);
-        if (params)
-                local->cont.create.params = dict_ref (params);
+    if (local->op != GF_FOP_CREATE && local->op != GF_FOP_MKNOD &&
+        local->op != GF_FOP_MKDIR)
+        return;
 
-        if (loc->parent)
-                local->cont.create.parent_ino = loc->parent->ino;
+    pre_op_count = AFR_COUNT(local->transaction.pre_op, priv->child_count);
+    failed_count = AFR_COUNT(local->transaction.failed_subvols,
+                             priv->child_count);
 
-	local->transaction.fop    = afr_create_wind;
-	local->transaction.done   = afr_create_done;
-	local->transaction.unwind = afr_create_unwind;
+    /* FOP succeeded on all bricks. */
+    if (pre_op_count == priv->child_count && !failed_count)
+        return;
 
-	afr_build_parent_loc (&local->transaction.parent_loc, loc);
+    /* FOP did not suceed on quorum no. of bricks. */
+    success_replies = alloca0(priv->child_count);
+    afr_fill_success_replies(local, priv, success_replies);
+    if (!afr_has_quorum(success_replies, this, NULL))
+        return;
 
-	local->transaction.main_frame = frame;
-	local->transaction.basename = AFR_BASENAME (loc->path);
+    if (priv->thin_arbiter_count) {
+        /*Mark new entry using ta file*/
+        local->is_new_entry = _gf_true;
+        return;
+    }
 
-	afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+    afr_mark_new_entry_changelog(frame, this);
 
-	op_ret = 0;
-out:
-	if (op_ret == -1) {
-		if (transaction_frame)
-			AFR_STACK_DESTROY (transaction_frame);
-		AFR_STACK_UNWIND (create, frame, op_ret, op_errno,
-                                  NULL, NULL, NULL, NULL, NULL);
-	}
-
-	return 0;
+    return;
 }
 
-/* }}} */
-
-/* {{{ mknod */
+/* {{{ create */
 
 int
-afr_mknod_unwind (call_frame_t *frame, xlator_t *this)
+afr_create_unwind(call_frame_t *frame, xlator_t *this)
 {
-	call_frame_t *main_frame = NULL;
-	afr_local_t  *local = NULL;
-
-        struct iatt *unwind_buf = NULL;
-
-	local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		if (local->transaction.main_frame) {
-			main_frame = local->transaction.main_frame;
-		}
-		local->transaction.main_frame = NULL;
-	}
-	UNLOCK (&frame->lock);
-
-	if (main_frame) {
-                if (local->cont.mknod.read_child_buf.ia_ino) {
-                        unwind_buf = &local->cont.mknod.read_child_buf;
-                } else {
-                        unwind_buf = &local->cont.mknod.buf;
-                }
-
-                unwind_buf->ia_ino = local->cont.mknod.ino;
-
-                local->cont.mknod.preparent.ia_ino  = local->cont.mknod.parent_ino;
-                local->cont.mknod.postparent.ia_ino = local->cont.mknod.parent_ino;
-
-		AFR_STACK_UNWIND (mknod, main_frame,
-                                  local->op_ret, local->op_errno,
-				  local->cont.mknod.inode,
-				  unwind_buf, &local->cont.mknod.preparent,
-                                  &local->cont.mknod.postparent);
-        }
+    call_frame_t *main_frame = NULL;
+    afr_local_t *local = NULL;
 
-	return 0;
-}
+    local = frame->local;
 
+    main_frame = afr_transaction_detach_fop_frame(frame);
+
+    if (!main_frame)
+        return 0;
+
+    AFR_STACK_UNWIND(create, main_frame, local->op_ret, local->op_errno,
+                     local->cont.create.fd, local->inode,
+                     &local->cont.dir_fop.buf, &local->cont.dir_fop.preparent,
+                     &local->cont.dir_fop.postparent, local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_mknod_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, inode_t *inode,
+afr_create_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
                     struct iatt *buf, struct iatt *preparent,
-                    struct iatt *postparent)
+                    struct iatt *postparent, dict_t *xdata)
 {
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
-
-	int call_count = -1;
-	int child_index = -1;
-
-	local = frame->local;
-	priv = this->private;
-
-	child_index = (long) cookie;
-
-	LOCK (&frame->lock);
-	{
-		if (afr_fop_failed (op_ret, op_errno))
-			afr_transaction_fop_failed (frame, this, child_index);
-		
-		if (op_ret != -1) {
-			local->op_ret = op_ret;
-
-			if (local->success_count == 0){
-				local->cont.mknod.buf   = *buf;
-				local->cont.mknod.ino   = 
-					afr_itransform (buf->ia_ino,
-							priv->child_count,
-							child_index);
-
-                                if (priv->read_child >= 0) {
-                                        afr_set_read_child (this, inode,
-                                                            priv->read_child);
-                                } else {
-                                        afr_set_read_child (this, inode,
-                                                            local->read_child_index);
-                                }
-			}
-
-                        if (child_index == local->first_up_child) {
-                                local->cont.mknod.ino =
-                                        afr_itransform (buf->ia_ino,
-                                                        priv->child_count,
-                                                        local->first_up_child);
-                        }
-
-                        if (child_index == local->read_child_index) {
-                                local->cont.mknod.read_child_buf = *buf;
-                                local->cont.mknod.preparent      = *preparent;
-                                local->cont.mknod.postparent     = *postparent;
-                        }
-                        
-			local->cont.mknod.inode = inode;
-
-			local->success_count++;
-		}
-
-		local->op_errno = op_errno;
-	}
-	UNLOCK (&frame->lock);
-
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0) {
-		local->transaction.unwind (frame, this);
-
-		local->transaction.resume (frame, this);
-	}
-	
-	return 0;
+    return __afr_dir_write_cbk(frame, cookie, this, op_ret, op_errno, buf,
+                               preparent, postparent, NULL, NULL, xdata);
 }
 
-
-int32_t
-afr_mknod_wind (call_frame_t *frame, xlator_t *this)
+int
+afr_create_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-
-	int call_count = -1;
-	int i = 0;
-
-	local = frame->local;
-	priv  = this->private;
-
-	call_count = afr_up_children_count (priv->child_count, local->child_up);
-
-	if (call_count == 0) {
-		local->transaction.resume (frame, this);
-		return 0;
-	}
-
-	local->call_count = call_count;
-
-	for (i = 0; i < priv->child_count; i++) {				
-		if (local->child_up[i]) {
-			STACK_WIND_COOKIE (frame, afr_mknod_wind_cbk, (void *) (long) i,
-					   priv->children[i], 
-					   priv->children[i]->fops->mknod,
-					   &local->loc, local->cont.mknod.mode,
-					   local->cont.mknod.dev,
-                                           local->cont.mknod.params);
-			if (!--call_count)
-				break;
-		}
-	}
-	
-	return 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    STACK_WIND_COOKIE(frame, afr_create_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->create, &local->loc,
+                      local->cont.create.flags, local->cont.create.mode,
+                      local->umask, local->cont.create.fd, local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_mknod_done (call_frame_t *frame, xlator_t *this)
+afr_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+           mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
-	afr_local_t * local = NULL;
-
-	local = frame->local;
-
-	local->transaction.unwind (frame, this);
-	AFR_STACK_DESTROY (frame);
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
+
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
+
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
+
+    loc_copy(&local->loc, loc);
+
+    local->fd_ctx = afr_fd_ctx_get(fd, this);
+    if (!local->fd_ctx)
+        goto out;
+
+    local->inode = inode_ref(loc->inode);
+    local->parent = inode_ref(loc->parent);
+
+    local->op = GF_FOP_CREATE;
+    local->cont.create.flags = flags;
+    local->fd_ctx->flags = flags;
+    local->cont.create.mode = mode;
+    local->cont.create.fd = fd_ref(fd);
+    local->umask = umask;
+
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
+
+    if (!local->xdata_req)
+        goto out;
+
+    local->transaction.wind = afr_create_wind;
+    local->transaction.unwind = afr_create_unwind;
+
+    ret = afr_build_parent_loc(&local->transaction.parent_loc, loc, &op_errno);
+    if (ret)
+        goto out;
+
+    local->transaction.main_frame = frame;
+    local->transaction.basename = AFR_BASENAME(loc->path);
+    ret = afr_transaction(transaction_frame, this, AFR_ENTRY_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
+out:
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	return 0;
+    AFR_STACK_UNWIND(create, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL,
+                     NULL);
+    return 0;
 }
 
+/* }}} */
+
+/* {{{ mknod */
 
 int
-afr_mknod (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, mode_t mode, dev_t dev, dict_t *params)
+afr_mknod_unwind(call_frame_t *frame, xlator_t *this)
 {
-	afr_private_t * priv  = NULL;
-	afr_local_t   * local = NULL;
-	call_frame_t  * transaction_frame = NULL;
-
-	int ret = -1;
-
-	int op_ret   = -1;
-	int op_errno = 0;
-
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
-
-	priv = this->private;
-
-	transaction_frame = copy_frame (frame);
-	if (!transaction_frame) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory.");
-		goto out;
-	}
-
-	ALLOC_OR_GOTO (local, afr_local_t, out);
+    call_frame_t *main_frame = NULL;
+    afr_local_t *local = NULL;
 
-	ret = AFR_LOCAL_INIT (local, priv);
-	if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-	}
+    local = frame->local;
 
-	transaction_frame->local = local;
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
+        return 0;
 
-	loc_copy (&local->loc, loc);
-
-        LOCK (&priv->read_child_lock);
-        {
-                local->read_child_index = (++priv->read_child_rr) 
-                        % (priv->child_count);
-        }
-        UNLOCK (&priv->read_child_lock);
-
-	local->cont.mknod.mode  = mode;
-	local->cont.mknod.dev   = dev;
-        if (params)
-                local->cont.mknod.params = dict_ref (params);
-
-        if (loc->parent)
-                local->cont.mknod.parent_ino = loc->parent->ino;
-
-	local->transaction.fop    = afr_mknod_wind;
-	local->transaction.done   = afr_mknod_done;
-	local->transaction.unwind = afr_mknod_unwind;
-
-	afr_build_parent_loc (&local->transaction.parent_loc, loc);
+    AFR_STACK_UNWIND(mknod, main_frame, local->op_ret, local->op_errno,
+                     local->inode, &local->cont.dir_fop.buf,
+                     &local->cont.dir_fop.preparent,
+                     &local->cont.dir_fop.postparent, local->xdata_rsp);
+    return 0;
+}
 
-	local->transaction.main_frame = frame;
-	local->transaction.basename = AFR_BASENAME (loc->path);
+int
+afr_mknod_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, inode_t *inode,
+                   struct iatt *buf, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
+{
+    return __afr_dir_write_cbk(frame, cookie, this, op_ret, op_errno, buf,
+                               preparent, postparent, NULL, NULL, xdata);
+}
 
-	afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+int
+afr_mknod_wind(call_frame_t *frame, xlator_t *this, int subvol)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    STACK_WIND_COOKIE(frame, afr_mknod_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->mknod, &local->loc,
+                      local->cont.mknod.mode, local->cont.mknod.dev,
+                      local->umask, local->xdata_req);
+    return 0;
+}
 
-	op_ret = 0;
+int
+afr_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          dev_t dev, mode_t umask, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
+
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
+
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
+
+    loc_copy(&local->loc, loc);
+    local->inode = inode_ref(loc->inode);
+    local->parent = inode_ref(loc->parent);
+
+    local->op = GF_FOP_MKNOD;
+    local->cont.mknod.mode = mode;
+    local->cont.mknod.dev = dev;
+    local->umask = umask;
+
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
+
+    if (!local->xdata_req)
+        goto out;
+
+    local->transaction.wind = afr_mknod_wind;
+    local->transaction.unwind = afr_mknod_unwind;
+
+    ret = afr_build_parent_loc(&local->transaction.parent_loc, loc, &op_errno);
+    if (ret)
+        goto out;
+
+    local->transaction.main_frame = frame;
+    local->transaction.basename = AFR_BASENAME(loc->path);
+    ret = afr_transaction(transaction_frame, this, AFR_ENTRY_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
 out:
-	if (op_ret == -1) {
-		if (transaction_frame)
-			AFR_STACK_DESTROY (transaction_frame);
-		AFR_STACK_UNWIND (mknod, frame, op_ret, op_errno,
-                                  NULL, NULL, NULL, NULL);
-	}
-
-	return 0;
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
+
+    AFR_STACK_UNWIND(mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
+    return 0;
 }
 
 /* }}} */
 
 /* {{{ mkdir */
 
-
 int
-afr_mkdir_unwind (call_frame_t *frame, xlator_t *this)
+afr_mkdir_unwind(call_frame_t *frame, xlator_t *this)
 {
-	call_frame_t *main_frame = NULL;
-	afr_local_t  *local = NULL;
-
-        struct iatt *unwind_buf = NULL;
-        
-	local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		if (local->transaction.main_frame) {
-			main_frame = local->transaction.main_frame;
-		}
-		local->transaction.main_frame = NULL;
-	}
-	UNLOCK (&frame->lock);
-
-	if (main_frame) {
-                if (local->cont.mkdir.read_child_buf.ia_ino) {
-                        unwind_buf = &local->cont.mkdir.read_child_buf;
-                } else {
-                        unwind_buf = &local->cont.mkdir.buf;
-                }
-
-                unwind_buf->ia_ino = local->cont.mkdir.ino;
-
-                local->cont.mkdir.preparent.ia_ino  = local->cont.mkdir.parent_ino;
-                local->cont.mkdir.postparent.ia_ino = local->cont.mkdir.parent_ino;
-
-		AFR_STACK_UNWIND (mkdir, main_frame,
-                                  local->op_ret, local->op_errno,
-				  local->cont.mkdir.inode,
-				  unwind_buf, &local->cont.mkdir.preparent,
-                        &local->cont.mkdir.postparent);
-        }
+    call_frame_t *main_frame = NULL;
+    afr_local_t *local = NULL;
 
-	return 0;
-}
+    local = frame->local;
 
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
+        return 0;
 
-int
-afr_mkdir_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, inode_t *inode,
-                    struct iatt *buf, struct iatt *preparent,
-                    struct iatt *postparent)
-{
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
-
-	int call_count = -1;
-	int child_index = -1;
-
-	local = frame->local;
-	priv = this->private;
-
-	child_index = (long) cookie;
-
-	LOCK (&frame->lock);
-	{
-		if (afr_fop_failed (op_ret, op_errno))
-			afr_transaction_fop_failed (frame, this, child_index);
-
-		if (op_ret != -1) {
-			local->op_ret           = op_ret;
-
-			if (local->success_count == 0) {
-				local->cont.mkdir.buf   = *buf;
-
-                                local->cont.mkdir.ino =
-                                        afr_itransform (buf->ia_ino,
-                                                        priv->child_count,
-                                                        child_index);
-
-                                if (priv->read_child >= 0) {
-                                        afr_set_read_child (this, inode,
-                                                            priv->read_child);
-                                } else {
-                                        afr_set_read_child (this, inode,
-                                                            local->read_child_index);
-                                }
-			}
-
-                        if (child_index == local->first_up_child) {
-                                local->cont.mkdir.ino =
-                                        afr_itransform (buf->ia_ino,
-                                                        priv->child_count,
-                                                        local->first_up_child);
-                        }
-                        
-                        if (child_index == local->read_child_index) {
-                                local->cont.mkdir.read_child_buf = *buf;
-                                local->cont.mkdir.preparent      = *preparent;
-                                local->cont.mkdir.postparent     = *postparent;
-                        }
-
-			local->cont.mkdir.inode = inode;
-
-			local->success_count++;
-		}
-
-		local->op_errno         = op_errno;
-	}
-	UNLOCK (&frame->lock);
-
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0) {
-		local->transaction.unwind (frame, this);
-
-		local->transaction.resume (frame, this);
-	}
-	
-	return 0;
+    AFR_STACK_UNWIND(mkdir, main_frame, local->op_ret, local->op_errno,
+                     local->inode, &local->cont.dir_fop.buf,
+                     &local->cont.dir_fop.preparent,
+                     &local->cont.dir_fop.postparent, local->xdata_rsp);
+    return 0;
 }
 
-
 int
-afr_mkdir_wind (call_frame_t *frame, xlator_t *this)
+afr_mkdir_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, inode_t *inode,
+                   struct iatt *buf, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	
-	int call_count = -1;
-	int i = 0;
-
-	local = frame->local;
-	priv  = this->private;
-
-	call_count = afr_up_children_count (priv->child_count, local->child_up);
-
-	if (call_count == 0) {
-		local->transaction.resume (frame, this);
-		return 0;
-	}
-
-	local->call_count = call_count;
-
-	for (i = 0; i < priv->child_count; i++) {				
-		if (local->child_up[i]) {
-			STACK_WIND_COOKIE (frame, afr_mkdir_wind_cbk,
-					   (void *) (long) i,	
-					   priv->children[i], 
-					   priv->children[i]->fops->mkdir,
-					   &local->loc, local->cont.mkdir.mode,
-                                           local->cont.mkdir.params);
-			if (!--call_count)
-				break;
-		}
-	}
-	
-	return 0;
+    return __afr_dir_write_cbk(frame, cookie, this, op_ret, op_errno, buf,
+                               preparent, postparent, NULL, NULL, xdata);
 }
 
-
 int
-afr_mkdir_done (call_frame_t *frame, xlator_t *this)
+afr_mkdir_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-	afr_local_t * local = NULL;
-
-	local = frame->local;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-	local->transaction.unwind (frame, this);
+    local = frame->local;
+    priv = this->private;
 
-	AFR_STACK_DESTROY (frame);
-
-	return 0;
+    STACK_WIND_COOKIE(frame, afr_mkdir_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->mkdir, &local->loc,
+                      local->cont.mkdir.mode, local->umask, local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_mkdir (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, mode_t mode, dict_t *params)
+afr_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          mode_t umask, dict_t *xdata)
 {
-	afr_private_t * priv  = NULL;
-	afr_local_t   * local = NULL;
-	call_frame_t  * transaction_frame = NULL;
-
-	int ret = -1;
-
-	int op_ret   = -1;
-	int op_errno = 0;
-
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
-
-	priv = this->private;
-
-	transaction_frame = copy_frame (frame);
-	if (!transaction_frame) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory.");
-		goto out;
-	}
-
-	ALLOC_OR_GOTO (local, afr_local_t, out);
-
-	ret = AFR_LOCAL_INIT (local, priv);
-	if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-	}
-
-	transaction_frame->local = local;
-
-	loc_copy (&local->loc, loc);
-
-        LOCK (&priv->read_child_lock);
-        {
-                local->read_child_index = (++priv->read_child_rr) 
-                        % (priv->child_count);
-        }
-        UNLOCK (&priv->read_child_lock);
-
-	local->cont.mkdir.mode  = mode;
-        if (params)
-                local->cont.mkdir.params = dict_ref (params);
-
-        if (loc->parent)
-                local->cont.mkdir.parent_ino = loc->parent->ino;
-
-	local->transaction.fop    = afr_mkdir_wind;
-	local->transaction.done   = afr_mkdir_done;
-	local->transaction.unwind = afr_mkdir_unwind;
-
-	afr_build_parent_loc (&local->transaction.parent_loc, loc);
-
-	local->transaction.main_frame = frame;
-	local->transaction.basename = AFR_BASENAME (loc->path);
-
-	afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
-
-	op_ret = 0;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
+
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
+
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
+
+    loc_copy(&local->loc, loc);
+    local->inode = inode_ref(loc->inode);
+    local->parent = inode_ref(loc->parent);
+
+    local->cont.mkdir.mode = mode;
+    local->umask = umask;
+
+    if (!xdata || !dict_get_sizen(xdata, "gfid-req")) {
+        op_errno = EPERM;
+        gf_msg_callingfn(this->name, GF_LOG_WARNING, op_errno,
+                         AFR_MSG_GFID_NULL,
+                         "mkdir: %s is received "
+                         "without gfid-req %p",
+                         loc->path, xdata);
+        goto out;
+    }
+
+    local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    if (!local->xdata_req) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    local->op = GF_FOP_MKDIR;
+    local->transaction.wind = afr_mkdir_wind;
+    local->transaction.unwind = afr_mkdir_unwind;
+
+    ret = afr_build_parent_loc(&local->transaction.parent_loc, loc, &op_errno);
+    if (ret)
+        goto out;
+
+    local->transaction.main_frame = frame;
+    local->transaction.basename = AFR_BASENAME(loc->path);
+    ret = afr_transaction(transaction_frame, this, AFR_ENTRY_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
 out:
-	if (op_ret == -1) {
-		if (transaction_frame)
-			AFR_STACK_DESTROY (transaction_frame);
-
-		AFR_STACK_UNWIND (mkdir, frame, op_ret, op_errno,
-                                  NULL, NULL, NULL, NULL);
-	}
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	return 0;
+    AFR_STACK_UNWIND(mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
+    return 0;
 }
 
 /* }}} */
 
 /* {{{ link */
 
-
-int
-afr_link_unwind (call_frame_t *frame, xlator_t *this)
-{
-	call_frame_t *main_frame = NULL;
-	afr_local_t  *local = NULL;
-
-        struct iatt *unwind_buf = NULL;
-
-	local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		if (local->transaction.main_frame) {
-			main_frame = local->transaction.main_frame;
-		}
-		local->transaction.main_frame = NULL;
-	}
-	UNLOCK (&frame->lock);
-
-	if (main_frame) {
-                if (local->cont.link.read_child_buf.ia_ino) {
-                        unwind_buf = &local->cont.link.read_child_buf;
-                } else {
-                        unwind_buf = &local->cont.link.buf;
-                }
-
-		unwind_buf->ia_ino = local->cont.link.ino;
-
-                local->cont.link.preparent.ia_ino  = local->cont.link.parent_ino;
-                local->cont.link.postparent.ia_ino = local->cont.link.parent_ino;
-
-		AFR_STACK_UNWIND (link, main_frame,
-                                  local->op_ret, local->op_errno, 
-				  local->cont.link.inode,
-				  unwind_buf, &local->cont.link.preparent,
-                                  &local->cont.link.postparent);
-	}
-
-	return 0;
-}
-
-
 int
-afr_link_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this, 
-		   int32_t op_ret, int32_t op_errno, inode_t *inode,
-                   struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent)
+afr_link_unwind(call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
-
-	int call_count = -1;
-	int child_index = -1;
-
-	local = frame->local;
-	priv = this->private;
-
-	child_index = (long) cookie;
-
-	LOCK (&frame->lock);
-	{
-		if (afr_fop_failed (op_ret, op_errno))
-			afr_transaction_fop_failed (frame, this, child_index);
-
-		if (op_ret != -1) {
-			local->op_ret   = op_ret;
-
-			if (local->success_count == 0) {
-				local->cont.link.buf        = *buf;
-
-                                if (priv->read_child >= 0) {
-                                        afr_set_read_child (this, inode,
-                                                            priv->read_child);
-                                } else {
-                                        afr_set_read_child (this, inode,
-                                                            local->read_child_index);
-                                }
-			}
-
-                        if (child_index == local->read_child_index) {
-                                local->cont.link.read_child_buf = *buf;
-                                local->cont.link.preparent      = *preparent;
-                                local->cont.link.postparent     = *postparent;
-                        }
+    call_frame_t *main_frame = NULL;
+    afr_local_t *local = NULL;
 
-			local->cont.link.inode    = inode;
+    local = frame->local;
 
-			local->success_count++;
-		}
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
+        return 0;
 
-		local->op_errno = op_errno;		
-	}
-	UNLOCK (&frame->lock);
-
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0) {
-		local->transaction.unwind (frame, this);
-
-		local->transaction.resume (frame, this);
-	}
-	
-	return 0;
+    AFR_STACK_UNWIND(link, main_frame, local->op_ret, local->op_errno,
+                     local->inode, &local->cont.dir_fop.buf,
+                     &local->cont.dir_fop.preparent,
+                     &local->cont.dir_fop.postparent, local->xdata_rsp);
+    return 0;
 }
 
-
 int
-afr_link_wind (call_frame_t *frame, xlator_t *this)
+afr_link_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, inode_t *inode,
+                  struct iatt *buf, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-
-	int call_count = -1;
-	int i = 0;
-
-	local = frame->local;
-	priv  = this->private;
-
-	call_count = afr_up_children_count (priv->child_count, local->child_up);
-
-	if (call_count == 0) {
-		local->transaction.resume (frame, this);
-		return 0;
-	}
-
-	local->call_count = call_count;
-
-	for (i = 0; i < priv->child_count; i++) {				
-		if (local->child_up[i]) {
-			STACK_WIND_COOKIE (frame, afr_link_wind_cbk, (void *) (long) i,
-					   priv->children[i], 
-					   priv->children[i]->fops->link,
-					   &local->loc,
-					   &local->newloc);
-			
-			if (!--call_count)
-				break;
-		}
-	}
-	
-	return 0;
+    return __afr_dir_write_cbk(frame, cookie, this, op_ret, op_errno, buf,
+                               preparent, postparent, NULL, NULL, xdata);
 }
 
-
 int
-afr_link_done (call_frame_t *frame, xlator_t *this)
+afr_link_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-	afr_local_t * local = frame->local;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-	local->transaction.unwind (frame, this);
+    local = frame->local;
+    priv = this->private;
 
-	AFR_STACK_DESTROY (frame);
-
-	return 0;
+    STACK_WIND_COOKIE(frame, afr_link_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->link, &local->loc,
+                      &local->newloc, local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_link (call_frame_t *frame, xlator_t *this,
-	  loc_t *oldloc, loc_t *newloc)
+afr_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+         dict_t *xdata)
 {
-	afr_private_t * priv  = NULL;
-	afr_local_t   * local = NULL;
-	call_frame_t  * transaction_frame = NULL;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
 
-	int ret = -1;
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-	int op_ret   = -1;
-	int op_errno = 0;
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
+    loc_copy(&local->loc, oldloc);
+    loc_copy(&local->newloc, newloc);
 
-	priv = this->private;
+    local->inode = inode_ref(oldloc->inode);
+    local->parent = inode_ref(newloc->parent);
 
-	transaction_frame = copy_frame (frame);
-	if (!transaction_frame) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory.");
-		goto out;
-	}
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
 
-	ALLOC_OR_GOTO (local, afr_local_t, out);
+    if (!local->xdata_req)
+        goto out;
 
-	ret = AFR_LOCAL_INIT (local, priv);
-	if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-	}
+    local->op = GF_FOP_LINK;
 
-	transaction_frame->local = local;
+    local->transaction.wind = afr_link_wind;
+    local->transaction.unwind = afr_link_unwind;
 
-	loc_copy (&local->loc,    oldloc);
-	loc_copy (&local->newloc, newloc);
-
-        LOCK (&priv->read_child_lock);
-        {
-                local->read_child_index = (++priv->read_child_rr) 
-                        % (priv->child_count);
-        }
-        UNLOCK (&priv->read_child_lock);
+    ret = afr_build_parent_loc(&local->transaction.parent_loc, newloc,
+                               &op_errno);
+    if (ret)
+        goto out;
 
-	local->cont.link.ino = oldloc->inode->ino;
+    local->transaction.main_frame = frame;
+    local->transaction.basename = AFR_BASENAME(newloc->path);
+    ret = afr_transaction(transaction_frame, this, AFR_ENTRY_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
 
-        if (oldloc->parent)
-                local->cont.link.parent_ino = newloc->parent->ino;
-
-	local->transaction.fop    = afr_link_wind;
-	local->transaction.done   = afr_link_done;
-	local->transaction.unwind = afr_link_unwind;
-
-	afr_build_parent_loc (&local->transaction.parent_loc, oldloc);
-
-	local->transaction.main_frame   = frame;
-	local->transaction.basename     = AFR_BASENAME (oldloc->path);
-	local->transaction.new_basename = AFR_BASENAME (newloc->path);
-
-	afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
-
-	op_ret = 0;
+    return 0;
 out:
-	if (op_ret == -1) {
-		if (transaction_frame)
-			AFR_STACK_DESTROY (transaction_frame);
-		AFR_STACK_UNWIND (link, frame, op_ret, op_errno,
-                                  NULL, NULL, NULL, NULL);
-	}
-
-	return 0;
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
+
+    AFR_STACK_UNWIND(link, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
+    return 0;
 }
 
 /* }}} */
 
 /* {{{ symlink */
 
-
 int
-afr_symlink_unwind (call_frame_t *frame, xlator_t *this)
+afr_symlink_unwind(call_frame_t *frame, xlator_t *this)
 {
-	call_frame_t *main_frame = NULL;
-	afr_local_t  *local = NULL;
-        
-        struct iatt *unwind_buf = NULL;
-        
-	local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		if (local->transaction.main_frame) {
-			main_frame = local->transaction.main_frame;
-		}
-		local->transaction.main_frame = NULL;
-	}
-	UNLOCK (&frame->lock);
-
-	if (main_frame) {
-                if (local->cont.symlink.read_child_buf.ia_ino) {
-                        unwind_buf = &local->cont.symlink.read_child_buf;
-                } else {
-                        unwind_buf = &local->cont.symlink.buf;
-                }
-
-                unwind_buf->ia_ino = local->cont.symlink.ino;
-
-                local->cont.symlink.preparent.ia_ino  = local->cont.symlink.parent_ino;
-                local->cont.symlink.postparent.ia_ino = local->cont.symlink.parent_ino;
-
-		AFR_STACK_UNWIND (symlink, main_frame,
-                                  local->op_ret, local->op_errno,
-				  local->cont.symlink.inode,
-				  unwind_buf, &local->cont.symlink.preparent,
-                                  &local->cont.symlink.postparent);
-        }
+    call_frame_t *main_frame = NULL;
+    afr_local_t *local = NULL;
 
-	return 0;
-}
+    local = frame->local;
 
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
+        return 0;
 
-int
-afr_symlink_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this, 
-		      int32_t op_ret, int32_t op_errno, inode_t *inode,
-                      struct iatt *buf, struct iatt *preparent,
-                      struct iatt *postparent)
-{
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
-
-	int call_count = -1;
-	int child_index = -1;
-
-	local = frame->local;
-	priv = this->private;
-
-	child_index = (long) cookie;
-
-	LOCK (&frame->lock);
-	{
-		if (afr_fop_failed (op_ret, op_errno))
-			afr_transaction_fop_failed (frame, this, child_index);
-		
-		if (op_ret != -1) {
-			local->op_ret   = op_ret;
-
-			if (local->success_count == 0) {
-				local->cont.symlink.buf        = *buf;
-				local->cont.symlink.ino = 
-					afr_itransform (buf->ia_ino, priv->child_count,
-							child_index);
-
-                                if (priv->read_child >= 0) {
-                                        afr_set_read_child (this, inode,
-                                                            priv->read_child);
-                                } else {
-                                        afr_set_read_child (this, inode,
-                                                            local->read_child_index);
-                                }
-			}
-
-                        if (child_index == local->first_up_child) {
-                                local->cont.symlink.ino =
-                                        afr_itransform (buf->ia_ino,
-                                                        priv->child_count,
-                                                        local->first_up_child);
-                        }
-
-                        if (child_index == local->read_child_index) {
-                                local->cont.symlink.read_child_buf = *buf;
-                                local->cont.symlink.preparent      = *preparent;
-                                local->cont.symlink.postparent     = *postparent;
-                        }
-
-			local->cont.symlink.inode    = inode;
-
-			local->success_count++;
-		}
-
-		local->op_errno = op_errno;
-	}
-	UNLOCK (&frame->lock);
-
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0) {
-		local->transaction.unwind (frame, this);
-
-		local->transaction.resume (frame, this);
-	}
-	
-	return 0;
+    AFR_STACK_UNWIND(symlink, main_frame, local->op_ret, local->op_errno,
+                     local->inode, &local->cont.dir_fop.buf,
+                     &local->cont.dir_fop.preparent,
+                     &local->cont.dir_fop.postparent, local->xdata_rsp);
+    return 0;
 }
 
-
 int
-afr_symlink_wind (call_frame_t *frame, xlator_t *this)
+afr_symlink_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, inode_t *inode,
+                     struct iatt *buf, struct iatt *preparent,
+                     struct iatt *postparent, dict_t *xdata)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-
-	int call_count = -1;
-	int i = 0;
-
-	local = frame->local;
-	priv = this->private;
-
-	call_count = afr_up_children_count (priv->child_count, local->child_up);
-
-	if (call_count == 0) {
-		local->transaction.resume (frame, this);
-		return 0;
-	}
-
-	local->call_count = call_count;
-
-	for (i = 0; i < priv->child_count; i++) {				
-		if (local->child_up[i]) {
-			STACK_WIND_COOKIE (frame, afr_symlink_wind_cbk,
-					   (void *) (long) i,	
-					   priv->children[i], 
-					   priv->children[i]->fops->symlink,
-					   local->cont.symlink.linkpath,
-					   &local->loc,
-                                           local->cont.symlink.params);
-
-			if (!--call_count)
-				break;
-
-		}
-	}
-	
-	return 0;
+    return __afr_dir_write_cbk(frame, cookie, this, op_ret, op_errno, buf,
+                               preparent, postparent, NULL, NULL, xdata);
 }
 
-
 int
-afr_symlink_done (call_frame_t *frame, xlator_t *this)
+afr_symlink_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-	afr_local_t * local = frame->local;
-
-	local->transaction.unwind (frame, this);
-
-	AFR_STACK_DESTROY (frame);
-	
-	return 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    STACK_WIND_COOKIE(frame, afr_symlink_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->symlink,
+                      local->cont.symlink.linkpath, &local->loc, local->umask,
+                      local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_symlink (call_frame_t *frame, xlator_t *this,
-	     const char *linkpath, loc_t *loc, dict_t *params)
+afr_symlink(call_frame_t *frame, xlator_t *this, const char *linkpath,
+            loc_t *loc, mode_t umask, dict_t *xdata)
 {
-	afr_private_t * priv  = NULL;
-	afr_local_t   * local = NULL;
-	call_frame_t  * transaction_frame = NULL;
-
-	int ret = -1;
-
-	int op_ret   = -1;
-	int op_errno = 0;
-
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
-
-	priv = this->private;
-
-	transaction_frame = copy_frame (frame);
-	if (!transaction_frame) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory.");
-		goto out;
-	}
-
-	ALLOC_OR_GOTO (local, afr_local_t, out);
-
-	ret = AFR_LOCAL_INIT (local, priv);
-	if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-	}
-
-	transaction_frame->local = local;
-	
-	loc_copy (&local->loc, loc);
-
-        LOCK (&priv->read_child_lock);
-        {
-                local->read_child_index = (++priv->read_child_rr) 
-                        % (priv->child_count);
-        }
-        UNLOCK (&priv->read_child_lock);
-
-	local->cont.symlink.linkpath = gf_strdup (linkpath);
-        if (params)
-                local->cont.symlink.params = dict_ref (params);
-
-        if (loc->parent)
-                local->cont.symlink.parent_ino = loc->parent->ino;
-
-	local->transaction.fop    = afr_symlink_wind;
-	local->transaction.done   = afr_symlink_done;
-	local->transaction.unwind = afr_symlink_unwind;
-
-	afr_build_parent_loc (&local->transaction.parent_loc, loc);
-
-	local->transaction.main_frame   = frame;
-	local->transaction.basename     = AFR_BASENAME (loc->path);
-
-	afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
-
-	op_ret = 0;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
+
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
+
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
+
+    loc_copy(&local->loc, loc);
+    local->inode = inode_ref(loc->inode);
+    local->parent = inode_ref(loc->parent);
+
+    local->cont.symlink.linkpath = gf_strdup(linkpath);
+    local->umask = umask;
+
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
+
+    if (!local->xdata_req)
+        goto out;
+
+    local->op = GF_FOP_SYMLINK;
+    local->transaction.wind = afr_symlink_wind;
+    local->transaction.unwind = afr_symlink_unwind;
+
+    ret = afr_build_parent_loc(&local->transaction.parent_loc, loc, &op_errno);
+    if (ret)
+        goto out;
+
+    local->transaction.main_frame = frame;
+    local->transaction.basename = AFR_BASENAME(loc->path);
+    ret = afr_transaction(transaction_frame, this, AFR_ENTRY_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
 out:
-	if (op_ret == -1) {
-		if (transaction_frame)
-			AFR_STACK_DESTROY (transaction_frame);
-		AFR_STACK_UNWIND (symlink, frame, op_ret, op_errno,
-                                  NULL, NULL, NULL, NULL);
-	}
-
-	return 0;
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
+
+    AFR_STACK_UNWIND(symlink, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                     NULL);
+    return 0;
 }
 
 /* }}} */
@@ -1351,230 +939,118 @@ out:
 /* {{{ rename */
 
 int
-afr_rename_unwind (call_frame_t *frame, xlator_t *this)
+afr_rename_unwind(call_frame_t *frame, xlator_t *this)
 {
-	call_frame_t *main_frame = NULL;
-	afr_local_t  *local = NULL;
-
-        struct iatt *unwind_buf = NULL;
-
-	local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		if (local->transaction.main_frame) {
-			main_frame = local->transaction.main_frame;
-		}
-		local->transaction.main_frame = NULL;
-	}
-	UNLOCK (&frame->lock);
-
-	if (main_frame) {
-                if (local->cont.rename.read_child_buf.ia_ino) {
-                        unwind_buf = &local->cont.rename.read_child_buf;
-                } else {
-                        unwind_buf = &local->cont.rename.buf;
-                }
-
-		unwind_buf->ia_ino = local->cont.rename.ino;
-
-                local->cont.rename.preoldparent.ia_ino  = local->cont.rename.oldparent_ino;
-                local->cont.rename.postoldparent.ia_ino = local->cont.rename.oldparent_ino;
-                local->cont.rename.prenewparent.ia_ino  = local->cont.rename.newparent_ino;
-                local->cont.rename.postnewparent.ia_ino = local->cont.rename.newparent_ino;
-
-		AFR_STACK_UNWIND (rename, main_frame,
-                                  local->op_ret, local->op_errno, 
-				  unwind_buf,
-                                  &local->cont.rename.preoldparent,
-                                  &local->cont.rename.postoldparent,
-                                  &local->cont.rename.prenewparent,
-                                  &local->cont.rename.postnewparent);
-	}
-
-	return 0;
-}
+    call_frame_t *main_frame = NULL;
+    afr_local_t *local = NULL;
 
+    local = frame->local;
 
-int
-afr_rename_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this, 
-		     int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                     struct iatt *preoldparent, struct iatt *postoldparent,
-                     struct iatt *prenewparent, struct iatt *postnewparent)
-{
-	afr_local_t *   local = NULL;
-
-	int call_count = -1;
-	int child_index = -1;
-
-	local = frame->local;
-
-	child_index = (long) cookie;
-
-	LOCK (&frame->lock);
-	{
-		if (afr_fop_failed (op_ret, op_errno) && op_errno != ENOTEMPTY)
-			afr_transaction_fop_failed (frame, this, child_index);
-
-		if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-		
-                                if (buf) {
-                                        local->cont.rename.buf = *buf;
-                                }
-
-                                local->success_count++;
-                        }
-
-                        if (child_index == local->read_child_index) {
-                                local->cont.rename.read_child_buf = *buf;
-
-                                local->cont.rename.preoldparent  = *preoldparent;
-                                local->cont.rename.postoldparent = *postoldparent;
-                                local->cont.rename.prenewparent  = *prenewparent;
-                                local->cont.rename.postnewparent = *postnewparent;
-                        }
-                }
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
+        return 0;
 
-		local->op_errno = op_errno;
-	}
-	UNLOCK (&frame->lock);
-
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0) {
-		local->transaction.unwind (frame, this);
-		local->transaction.resume (frame, this);
-	}
-	
-	return 0;
+    AFR_STACK_UNWIND(rename, main_frame, local->op_ret, local->op_errno,
+                     &local->cont.dir_fop.buf, &local->cont.dir_fop.preparent,
+                     &local->cont.dir_fop.postparent,
+                     &local->cont.dir_fop.prenewparent,
+                     &local->cont.dir_fop.postnewparent, local->xdata_rsp);
+    return 0;
 }
 
-
-int32_t
-afr_rename_wind (call_frame_t *frame, xlator_t *this)
+int
+afr_rename_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                    struct iatt *preoldparent, struct iatt *postoldparent,
+                    struct iatt *prenewparent, struct iatt *postnewparent,
+                    dict_t *xdata)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-
-	int call_count = -1;
-	int i = 0;
-
-	local = frame->local;
-	priv = this->private;
-
-	call_count = afr_up_children_count (priv->child_count, local->child_up);
-
-	if (call_count == 0) {
-		local->transaction.resume (frame, this);
-		return 0;
-	}
-
-	local->call_count = call_count;
-
-	for (i = 0; i < priv->child_count; i++) {				
-		if (local->child_up[i]) {
-			STACK_WIND_COOKIE (frame, afr_rename_wind_cbk, 
-					   (void *) (long) i,	
-					   priv->children[i], 
-					   priv->children[i]->fops->rename,
-					   &local->loc,
-					   &local->newloc);
-			if (!--call_count)
-				break;
-		}
-	}
-	
-	return 0;
+    return __afr_dir_write_cbk(frame, cookie, this, op_ret, op_errno, buf,
+                               preoldparent, postoldparent, prenewparent,
+                               postnewparent, xdata);
 }
 
-
 int
-afr_rename_done (call_frame_t *frame, xlator_t *this)
+afr_rename_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-	afr_local_t * local = frame->local;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-	local->transaction.unwind (frame, this);
+    local = frame->local;
+    priv = this->private;
 
-	AFR_STACK_DESTROY (frame);
-	
-	return 0;
+    STACK_WIND_COOKIE(frame, afr_rename_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->rename, &local->loc,
+                      &local->newloc, local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_rename (call_frame_t *frame, xlator_t *this,
-	    loc_t *oldloc, loc_t *newloc)
+afr_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+           dict_t *xdata)
 {
-	afr_private_t * priv  = NULL;
-	afr_local_t   * local = NULL;
-	call_frame_t  * transaction_frame = NULL;
-
-	int ret = -1;
-
-	int op_ret   = -1;
-	int op_errno = 0;
-
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
-
-	priv = this->private;
-
-	transaction_frame = copy_frame (frame);
-	if (!transaction_frame) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory.");
-		goto out;
-	}
-
-	ALLOC_OR_GOTO (local, afr_local_t, out);
-
-	ret = AFR_LOCAL_INIT (local, priv);
-	if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-	}
-
-	transaction_frame->local = local;
-
-	loc_copy (&local->loc,    oldloc);
-	loc_copy (&local->newloc, newloc);
-
-        local->read_child_index = afr_read_child (this, oldloc->inode);
-
-	local->cont.rename.ino = oldloc->inode->ino;
-
-        if (oldloc->parent)
-                local->cont.rename.oldparent_ino = oldloc->parent->ino;
-        if (newloc->parent)
-                local->cont.rename.newparent_ino = newloc->parent->ino;
-
-	local->transaction.fop    = afr_rename_wind;
-	local->transaction.done   = afr_rename_done;
-	local->transaction.unwind = afr_rename_unwind;
-
-	afr_build_parent_loc (&local->transaction.parent_loc, oldloc);
-	afr_build_parent_loc (&local->transaction.new_parent_loc, newloc);
-
-	local->transaction.main_frame   = frame;
-	local->transaction.basename     = AFR_BASENAME (oldloc->path);
-	local->transaction.new_basename = AFR_BASENAME (newloc->path);
-
-	afr_transaction (transaction_frame, this, AFR_ENTRY_RENAME_TRANSACTION);
-
-	op_ret = 0;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
+
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
+
+    loc_copy(&local->loc, oldloc);
+    loc_copy(&local->newloc, newloc);
+
+    local->inode = inode_ref(oldloc->inode);
+    local->parent = inode_ref(oldloc->parent);
+    local->parent2 = inode_ref(newloc->parent);
+
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
+
+    if (!local->xdata_req)
+        goto out;
+
+    local->op = GF_FOP_RENAME;
+    local->transaction.wind = afr_rename_wind;
+    local->transaction.unwind = afr_rename_unwind;
+
+    ret = afr_build_parent_loc(&local->transaction.parent_loc, oldloc,
+                               &op_errno);
+    if (ret)
+        goto out;
+    ret = afr_build_parent_loc(&local->transaction.new_parent_loc, newloc,
+                               &op_errno);
+    if (ret)
+        goto out;
+
+    local->transaction.main_frame = frame;
+    local->transaction.basename = AFR_BASENAME(oldloc->path);
+    local->transaction.new_basename = AFR_BASENAME(newloc->path);
+    ret = afr_transaction(transaction_frame, this,
+                          AFR_ENTRY_RENAME_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
 out:
-	if (op_ret == -1) {
-		if (transaction_frame)
-			AFR_STACK_DESTROY (transaction_frame);
-
-		AFR_STACK_UNWIND (rename, frame, op_ret, op_errno,
-                                  NULL, NULL, NULL, NULL, NULL);
-	}
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	return 0;
+    AFR_STACK_UNWIND(rename, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL,
+                     NULL);
+    return 0;
 }
 
 /* }}} */
@@ -1582,412 +1058,205 @@ out:
 /* {{{ unlink */
 
 int
-afr_unlink_unwind (call_frame_t *frame, xlator_t *this)
-{
-	call_frame_t *main_frame = NULL;
-	afr_local_t  *local = NULL;
-
-	local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		if (local->transaction.main_frame) {
-			main_frame = local->transaction.main_frame;
-		}
-		local->transaction.main_frame = NULL;
-	}
-	UNLOCK (&frame->lock);
-
-	if (main_frame) {
-                local->cont.unlink.preparent.ia_ino  = local->cont.unlink.parent_ino;
-                local->cont.unlink.postparent.ia_ino = local->cont.unlink.parent_ino;
-
-		AFR_STACK_UNWIND (unlink, main_frame,
-                                  local->op_ret, local->op_errno,
-                                  &local->cont.unlink.preparent,
-                                  &local->cont.unlink.postparent);
-        }
-
-	return 0;
-}
-
-
-int
-afr_unlink_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this, 
-		     int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                     struct iatt *postparent)
+afr_unlink_unwind(call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
-
-	int call_count  = -1;
-	int child_index = (long) cookie;
-
-	local = frame->local;
-	priv  = this->private;
-
-	LOCK (&frame->lock);
-	{
-                if (child_index == local->read_child_index) {
-                        local->read_child_returned = _gf_true;
-                }
-
-		if (afr_fop_failed (op_ret, op_errno))
-			afr_transaction_fop_failed (frame, this, child_index);
+    call_frame_t *main_frame = NULL;
+    afr_local_t *local = NULL;
 
-		if (op_ret != -1) {
-			if (local->success_count == 0) {
-				local->op_ret   = op_ret;
-                                local->cont.unlink.preparent  = *preparent;
-                                local->cont.unlink.postparent = *postparent;
-			}
+    local = frame->local;
 
-                        if (child_index == local->read_child_index) {
-                                local->cont.unlink.preparent  = *preparent;
-                                local->cont.unlink.postparent = *postparent;
-                        }
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
+        return 0;
 
-			local->success_count++;
-		}
-
-		local->op_errno = op_errno;
-	}
-	UNLOCK (&frame->lock);
-
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0) {
-		local->transaction.unwind (frame, this);
-
-		local->transaction.resume (frame, this);
-	}
-	
-	return 0;
+    AFR_STACK_UNWIND(unlink, main_frame, local->op_ret, local->op_errno,
+                     &local->cont.dir_fop.preparent,
+                     &local->cont.dir_fop.postparent, local->xdata_rsp);
+    return 0;
 }
 
-
-int32_t
-afr_unlink_wind (call_frame_t *frame, xlator_t *this)
+int
+afr_unlink_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-
-	int call_count = -1;
-	int i = 0;
-
-	local = frame->local;
-	priv  = this->private;
-
-	call_count = afr_up_children_count (priv->child_count, local->child_up);
-
-	if (call_count == 0) {
-		local->transaction.resume (frame, this);
-		return 0;
-	}
-
-	local->call_count = call_count;
-
-	for (i = 0; i < priv->child_count; i++) {				
-		if (local->child_up[i]) {
-			STACK_WIND_COOKIE (frame, afr_unlink_wind_cbk,	
-					   (void *) (long) i,
-					   priv->children[i], 
-					   priv->children[i]->fops->unlink,
-					   &local->loc);
-			
-			if (!--call_count)
-				break;
-		}
-	}
-	
-	return 0;
+    return __afr_dir_write_cbk(frame, cookie, this, op_ret, op_errno, NULL,
+                               preparent, postparent, NULL, NULL, xdata);
 }
 
-
-int32_t
-afr_unlink_done (call_frame_t *frame, xlator_t *this)
+int
+afr_unlink_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-	afr_local_t * local = frame->local;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-	local->transaction.unwind (frame, this);
+    local = frame->local;
+    priv = this->private;
 
-	AFR_STACK_DESTROY (frame);
-	
-	return 0;
+    STACK_WIND_COOKIE(frame, afr_unlink_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->unlink, &local->loc,
+                      local->xflag, local->xdata_req);
+    return 0;
 }
 
-
-int32_t
-afr_unlink (call_frame_t *frame, xlator_t *this,
-	    loc_t *loc)
+int
+afr_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+           dict_t *xdata)
 {
-	afr_private_t * priv  = NULL;
-	afr_local_t   * local = NULL;
-	call_frame_t  * transaction_frame = NULL;
-
-	int ret = -1;
-
-	int op_ret   = -1;
-	int op_errno = 0;
-
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
-
-	priv = this->private;
-
-	transaction_frame = copy_frame (frame);
-	if (!transaction_frame) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory.");
-		goto out;
-	}
-
-	ALLOC_OR_GOTO (local, afr_local_t, out);
-
-	ret = AFR_LOCAL_INIT (local, priv);
-	if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-	}
-
-	transaction_frame->local = local;
-
-	loc_copy (&local->loc, loc);
-
-        if (loc->parent)
-                local->cont.unlink.parent_ino = loc->parent->ino;
-
-	local->transaction.fop    = afr_unlink_wind;
-	local->transaction.done   = afr_unlink_done;
-	local->transaction.unwind = afr_unlink_unwind;
-
-	afr_build_parent_loc (&local->transaction.parent_loc, loc);
-
-	local->transaction.main_frame = frame;
-	local->transaction.basename = AFR_BASENAME (loc->path);
-
-	afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
-
-	op_ret = 0;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
+
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
+
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
+
+    loc_copy(&local->loc, loc);
+    local->xflag = xflag;
+
+    local->inode = inode_ref(loc->inode);
+    local->parent = inode_ref(loc->parent);
+
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
+
+    if (!local->xdata_req)
+        goto out;
+
+    local->op = GF_FOP_UNLINK;
+    local->transaction.wind = afr_unlink_wind;
+    local->transaction.unwind = afr_unlink_unwind;
+
+    ret = afr_build_parent_loc(&local->transaction.parent_loc, loc, &op_errno);
+    if (ret)
+        goto out;
+
+    local->transaction.main_frame = frame;
+    local->transaction.basename = AFR_BASENAME(loc->path);
+    ret = afr_transaction(transaction_frame, this, AFR_ENTRY_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
 out:
-	if (op_ret == -1) {
-		if (transaction_frame)
-			AFR_STACK_DESTROY (transaction_frame);
-		AFR_STACK_UNWIND (unlink, frame, op_ret, op_errno,
-                                  NULL, NULL);
-	}
-
-	return 0;
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
+
+    AFR_STACK_UNWIND(unlink, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
 /* }}} */
 
 /* {{{ rmdir */
 
-
-
 int
-afr_rmdir_unwind (call_frame_t *frame, xlator_t *this)
+afr_rmdir_unwind(call_frame_t *frame, xlator_t *this)
 {
-	call_frame_t *main_frame = NULL;
-	afr_local_t  *local = NULL;
-
-	local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		if (local->transaction.main_frame) {
-			main_frame = local->transaction.main_frame;
-		}
-		local->transaction.main_frame = NULL;
-	}
-	UNLOCK (&frame->lock);
-
-	if (main_frame) {
-                local->cont.rmdir.preparent.ia_ino  = local->cont.rmdir.parent_ino;
-                local->cont.rmdir.postparent.ia_ino = local->cont.rmdir.parent_ino;
-
-		AFR_STACK_UNWIND (rmdir, main_frame,
-                                  local->op_ret, local->op_errno,
-                                  &local->cont.rmdir.preparent,
-                                  &local->cont.rmdir.postparent);
-        }
-
-	return 0;
-}
-
-
-int
-afr_rmdir_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this, 
-		    int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                    struct iatt *postparent)
-{
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
-
-	int call_count  = -1;
-	int child_index = (long) cookie;
-        int read_child  = 0;
+    call_frame_t *main_frame = NULL;
+    afr_local_t *local = NULL;
 
-	local = frame->local;
-	priv = this->private;
+    local = frame->local;
 
-	LOCK (&frame->lock);
-	{
-                if (child_index == read_child) {
-                        local->read_child_returned = _gf_true;
-                }
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
+        return 0;
 
-		if (afr_fop_failed (op_ret, op_errno) && (op_errno != ENOTEMPTY))
-			afr_transaction_fop_failed (frame, this, child_index);
-
-		if (op_ret != -1) {
-			if (local->success_count == 0) {
-				local->op_ret = op_ret;
-                                local->cont.rmdir.preparent  = *preparent;
-                                local->cont.rmdir.postparent = *postparent;
-
-			}
-
-                        if (child_index == read_child) {
-                                local->cont.rmdir.preparent  = *preparent;
-                                local->cont.rmdir.postparent = *postparent;
-                        }
-
-			local->success_count++;
-		}
-
-		local->op_errno = op_errno;
-	}
-	UNLOCK (&frame->lock);
-
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0) {
-		local->transaction.unwind (frame, this);
-		local->transaction.resume (frame, this);
-	}
-	
-	return 0;
+    AFR_STACK_UNWIND(rmdir, main_frame, local->op_ret, local->op_errno,
+                     &local->cont.dir_fop.preparent,
+                     &local->cont.dir_fop.postparent, local->xdata_rsp);
+    return 0;
 }
 
-
 int
-afr_rmdir_wind (call_frame_t *frame, xlator_t *this)
+afr_rmdir_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-
-	int call_count = -1;
-	int i = 0;
-
-	local = frame->local;
-	priv  = this->private;
-
-	call_count = afr_up_children_count (priv->child_count, local->child_up);
-
-	if (call_count == 0) {
-		local->transaction.resume (frame, this);
-		return 0;
-	}
-
-	local->call_count = call_count;
-
-	for (i = 0; i < priv->child_count; i++) {				
-		if (local->child_up[i]) {
-			STACK_WIND_COOKIE (frame, afr_rmdir_wind_cbk,	
-					   (void *) (long) i,
-					   priv->children[i], 
-					   priv->children[i]->fops->rmdir,
-					   &local->loc, local->cont.rmdir.flags);
-
-			if (!--call_count)
-				break;
-		}
-	}
-	
-	return 0;
+    return __afr_dir_write_cbk(frame, cookie, this, op_ret, op_errno, NULL,
+                               preparent, postparent, NULL, NULL, xdata);
 }
 
-
 int
-afr_rmdir_done (call_frame_t *frame, xlator_t *this)
+afr_rmdir_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-	afr_local_t * local = frame->local;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-	local->transaction.unwind (frame, this);
+    local = frame->local;
+    priv = this->private;
 
-	AFR_STACK_DESTROY (frame);
-	
-	return 0;
+    STACK_WIND_COOKIE(frame, afr_rmdir_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->rmdir, &local->loc,
+                      local->cont.rmdir.flags, local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_rmdir (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, int flags)
+afr_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+          dict_t *xdata)
 {
-	afr_private_t * priv  = NULL;
-	afr_local_t   * local = NULL;
-	call_frame_t  * transaction_frame = NULL;
-	
-	int ret = -1;
-
-	int op_ret   = -1;
-	int op_errno = 0;
-
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
-
-	priv = this->private;
-
-	transaction_frame = copy_frame (frame);
-	if (!transaction_frame) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory.");
-		goto out;
-	}
-
-	ALLOC_OR_GOTO (local, afr_local_t, out);
-
-	ret = AFR_LOCAL_INIT (local, priv);
-	if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-	}
-
-	transaction_frame->local = local;
-
-        local->cont.rmdir.flags = flags;
-	loc_copy (&local->loc, loc);
-
-        if (loc->parent)
-                local->cont.rmdir.parent_ino = loc->parent->ino;
-
-	local->transaction.fop    = afr_rmdir_wind;
-	local->transaction.done   = afr_rmdir_done;
-	local->transaction.unwind = afr_rmdir_unwind;
-
-	afr_build_parent_loc (&local->transaction.parent_loc, loc);
-
-	local->transaction.main_frame = frame;
-	local->transaction.basename = AFR_BASENAME (loc->path);
-
-	afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
-
-	op_ret = 0;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
+
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
+
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
+
+    loc_copy(&local->loc, loc);
+    local->inode = inode_ref(loc->inode);
+    local->parent = inode_ref(loc->parent);
+
+    local->cont.rmdir.flags = flags;
+
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
+
+    if (!local->xdata_req)
+        goto out;
+
+    local->op = GF_FOP_RMDIR;
+    local->transaction.wind = afr_rmdir_wind;
+    local->transaction.unwind = afr_rmdir_unwind;
+
+    ret = afr_build_parent_loc(&local->transaction.parent_loc, loc, &op_errno);
+    if (ret)
+        goto out;
+
+    local->transaction.main_frame = frame;
+    local->transaction.basename = AFR_BASENAME(loc->path);
+    ret = afr_transaction(transaction_frame, this, AFR_ENTRY_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
 out:
-	if (op_ret == -1) {
-		if (transaction_frame)
-			AFR_STACK_DESTROY (transaction_frame);
-		AFR_STACK_UNWIND (rmdir, frame, op_ret, op_errno,
-                                  NULL, NULL);
-	}
-
-	return 0;
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
+
+    AFR_STACK_UNWIND(rmdir, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
 /* }}} */
-
diff --git a/xlators/cluster/afr/src/afr-dir-write.h b/xlators/cluster/afr/src/afr-dir-write.h
index e589efa3794..1d88c3b9b26 100644
--- a/xlators/cluster/afr/src/afr-dir-write.h
+++ b/xlators/cluster/afr/src/afr-dir-write.h
@@ -1,60 +1,46 @@
 /*
-   Copyright (c) 2007-2010 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __DIR_WRITE_H__
 #define __DIR_WRITE_H__
 
 int32_t
-afr_create (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, int32_t flags, mode_t mode,
-            fd_t *fd, dict_t *params);
+afr_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+           mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata);
 
 int32_t
-afr_mknod (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, mode_t mode, dev_t dev, dict_t *params);
+afr_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          dev_t dev, mode_t umask, dict_t *xdata);
 
 int32_t
-afr_mkdir (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, mode_t mode, dict_t *params);
+afr_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          mode_t umask, dict_t *xdata);
 
 int32_t
-afr_unlink (call_frame_t *frame, xlator_t *this,
-	    loc_t *loc);
+afr_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+           dict_t *xdata);
 
 int32_t
-afr_rmdir (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, int flags);
+afr_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+          dict_t *xdata);
 
 int32_t
-afr_link (call_frame_t *frame, xlator_t *this,
-	  loc_t *oldloc, loc_t *newloc);
+afr_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+         dict_t *xdata);
 
 int32_t
-afr_rename (call_frame_t *frame, xlator_t *this,
-	    loc_t *oldloc, loc_t *newloc);
+afr_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+           dict_t *xdata);
 
 int
-afr_symlink (call_frame_t *frame, xlator_t *this,
-	     const char *linkpath, loc_t *oldloc, dict_t *params);
-
-int32_t
-afr_setdents (call_frame_t *frame, xlator_t *this,
-	      fd_t *fd, int32_t flags, dir_entry_t *entries, int32_t count);
+afr_symlink(call_frame_t *frame, xlator_t *this, const char *linkpath,
+            loc_t *oldloc, mode_t umask, dict_t *params);
 
 #endif /* __DIR_WRITE_H__ */
diff --git a/xlators/cluster/afr/src/afr-inode-read.c b/xlators/cluster/afr/src/afr-inode-read.c
index ce6a6a363ed..c5521704de2 100644
--- a/xlators/cluster/afr/src/afr-inode-read.c
+++ b/xlators/cluster/afr/src/afr-inode-read.c
@@ -1,22 +1,12 @@
 /*
-   Copyright (c) 2007-2010 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
 
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
 
 #include <libgen.h>
 #include <unistd.h>
@@ -25,931 +15,1880 @@
 #include <stdlib.h>
 #include <signal.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "afr.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "stack.h"
-#include "list.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include "compat-errno.h"
-#include "compat.h"
-
+#include <glusterfs/glusterfs.h>
 #include "afr.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/list.h>
+#include <glusterfs/byte-order.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/quota-common-utils.h>
+
+#include "afr-transaction.h"
+#include "afr-messages.h"
 
-
-/**
- * Common algorithm for inode read calls:
- *
- * - Try the fop on the first child that is up
- * - if we have failed due to ENOTCONN:
- *     try the next child
- *
- * Applicable to: access, stat, fstat, readlink, getxattr
- */
+/*
+ * Quota size xattrs are not maintained by afr. There is a
+ * possibility that they differ even when both the directory changelog xattrs
+ * suggest everything is fine. So if there is at least one 'source' check among
+ * the sources which has the maximum quota size. Otherwise check among all the
+ * available ones for maximum quota size. This way if there is a source and
+ * stale copies it always votes for the 'source'.
+ * */
+
+int
+afr_handle_quota_size(call_frame_t *frame, xlator_t *this)
+{
+    unsigned char *readable = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    struct afr_reply *replies = NULL;
+    int i = 0;
+    int ret = 0;
+    quota_meta_t size = {
+        0,
+    };
+    quota_meta_t max_size = {
+        0,
+    };
+    int readable_cnt = 0;
+    int read_subvol = -1;
+
+    local = frame->local;
+    priv = this->private;
+    replies = local->replies;
+
+    readable = alloca0(priv->child_count);
+
+    afr_inode_read_subvol_get(local->inode, this, readable, 0, 0);
+
+    readable_cnt = AFR_COUNT(readable, priv->child_count);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret == -1)
+            continue;
+        if (readable_cnt && !readable[i])
+            continue;
+        if (!replies[i].xdata)
+            continue;
+        ret = quota_dict_get_meta(replies[i].xdata, QUOTA_SIZE_KEY,
+                                  SLEN(QUOTA_SIZE_KEY), &size);
+        if (ret == -1)
+            continue;
+        if (read_subvol == -1)
+            read_subvol = i;
+        if (size.size > max_size.size ||
+            (size.file_count + size.dir_count) >
+                (max_size.file_count + max_size.dir_count))
+            read_subvol = i;
+
+        if (size.size > max_size.size)
+            max_size.size = size.size;
+        if (size.file_count > max_size.file_count)
+            max_size.file_count = size.file_count;
+        if (size.dir_count > max_size.dir_count)
+            max_size.dir_count = size.dir_count;
+    }
+
+    if (max_size.size == 0 && max_size.file_count == 0 &&
+        max_size.dir_count == 0)
+        return read_subvol;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret == -1)
+            continue;
+        if (readable_cnt && !readable[i])
+            continue;
+        if (!replies[i].xdata)
+            continue;
+        quota_dict_set_meta(replies[i].xdata, QUOTA_SIZE_KEY, &max_size,
+                            IA_IFDIR);
+    }
+
+    return read_subvol;
+}
 
 /* {{{ access */
 
-int32_t
-afr_access_cbk (call_frame_t *frame, void *cookie,
-		xlator_t *this, int32_t op_ret, int32_t op_errno)
+int
+afr_access_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, dict_t *xdata)
 {
-	afr_private_t * priv     = NULL;
-	afr_local_t *   local    = NULL;
-	xlator_t **     children = NULL;
+    afr_local_t *local = NULL;
 
-	int unwind     = 1;
-	int last_tried = -1;
-	int this_try = -1;
-        int read_child = -1;
+    local = frame->local;
 
-	priv     = this->private;
-	children = priv->children;
+    if (op_ret < 0) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
 
-	local = frame->local;
+        afr_read_txn_continue(frame, this, (long)cookie);
+        return 0;
+    }
 
-        read_child = (long) cookie;
+    AFR_STACK_UNWIND(access, frame, op_ret, op_errno, xdata);
 
-	if (op_ret == -1) {
-        retry:
-		last_tried = local->cont.access.last_tried;
+    return 0;
+}
 
-		if (all_tried (last_tried, priv->child_count)) {
-			goto out;
-		}
-		this_try    = ++local->cont.access.last_tried;
+int
+afr_access_wind(call_frame_t *frame, xlator_t *this, int subvol)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
 
-                if (this_try == read_child) {
-                        goto retry;
-                }
+    priv = this->private;
+    local = frame->local;
 
-		unwind = 0;
+    if (subvol == -1) {
+        AFR_STACK_UNWIND(access, frame, local->op_ret, local->op_errno, 0);
+        return 0;
+    }
 
-		STACK_WIND_COOKIE (frame, afr_access_cbk,
-				   (void *) (long) read_child,
-				   children[this_try],
-				   children[this_try]->fops->access,
-				   &local->loc, local->cont.access.mask);
-	}
+    STACK_WIND_COOKIE(frame, afr_access_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->access, &local->loc,
+                      local->cont.access.mask, local->xdata_req);
+    return 0;
+}
 
+int
+afr_access(call_frame_t *frame, xlator_t *this, loc_t *loc, int mask,
+           dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    int op_errno = 0;
+
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
+
+    local->op = GF_FOP_ACCESS;
+    loc_copy(&local->loc, loc);
+    local->cont.access.mask = mask;
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
+
+    afr_read_txn(frame, this, loc->inode, afr_access_wind,
+                 AFR_METADATA_TRANSACTION);
+
+    return 0;
 out:
-	if (unwind) {
-		AFR_STACK_UNWIND (access, frame, op_ret, op_errno);
-	}
+    AFR_STACK_UNWIND(access, frame, -1, op_errno, NULL);
 
-	return 0;
+    return 0;
 }
 
+/* }}} */
 
-int32_t
-afr_access (call_frame_t *frame, xlator_t *this,
-	    loc_t *loc, int32_t mask)
-{
-	afr_private_t * priv       = NULL;
-	xlator_t **     children   = NULL;
-	int             call_child = 0;
-	afr_local_t     *local     = NULL;
+/* {{{ stat */
 
-        int32_t         read_child = -1;
+int
+afr_stat_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, struct iatt *buf, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
 
+    local = frame->local;
 
-	int32_t op_ret   = -1;
-	int32_t op_errno = 0;
+    if (op_ret < 0) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
 
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
+        afr_read_txn_continue(frame, this, (long)cookie);
+        return 0;
+    }
 
-	priv     = this->private;
-	VALIDATE_OR_GOTO (priv->children, out);
+    AFR_STACK_UNWIND(stat, frame, op_ret, op_errno, buf, xdata);
 
-	children = priv->children;
+    return 0;
+}
 
-	ALLOC_OR_GOTO (local, afr_local_t, out);
+int
+afr_stat_wind(call_frame_t *frame, xlator_t *this, int subvol)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
 
-        read_child = afr_read_child (this, loc->inode);
+    priv = this->private;
+    local = frame->local;
 
-        if ((read_child >= 0) && (priv->child_up[read_child])) {
-                call_child = read_child;
+    if (subvol == -1) {
+        AFR_STACK_UNWIND(stat, frame, local->op_ret, local->op_errno, 0, 0);
+        return 0;
+    }
 
-                local->cont.access.last_tried = -1;
+    STACK_WIND_COOKIE(
+        frame, afr_stat_cbk, (void *)(long)subvol, priv->children[subvol],
+        priv->children[subvol]->fops->stat, &local->loc, local->xdata_req);
+    return 0;
+}
 
-        } else {
-                call_child = afr_first_up_child (priv);
-                if (call_child == -1) {
-                        op_errno = ENOTCONN;
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "no child is up");
-                        goto out;
-                }
+int
+afr_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    int op_errno = 0;
 
-                local->cont.access.last_tried = call_child;
-        }
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-	loc_copy (&local->loc, loc);
-	local->cont.access.mask       = mask;
+    local->op = GF_FOP_STAT;
+    loc_copy(&local->loc, loc);
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
 
-	STACK_WIND_COOKIE (frame, afr_access_cbk,
-			   (void *) (long) call_child,
-			   children[call_child], children[call_child]->fops->access,
-			   loc, mask);
+    afr_read_txn(frame, this, loc->inode, afr_stat_wind, AFR_DATA_TRANSACTION);
 
-	op_ret = 0;
+    return 0;
 out:
-	if (op_ret == -1) {
-		AFR_STACK_UNWIND (access, frame, op_ret, op_errno);
-	}
-	return 0;
-}
+    AFR_STACK_UNWIND(stat, frame, -1, op_errno, NULL, NULL);
 
+    return 0;
+}
 
 /* }}} */
 
-/* {{{ stat */
+/* {{{ fstat */
 
-int32_t
-afr_stat_cbk (call_frame_t *frame, void *cookie,
-	      xlator_t *this, int32_t op_ret, int32_t op_errno,
-	      struct iatt *buf)
+int
+afr_fstat_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
-	afr_private_t * priv     = NULL;
-	afr_local_t *   local    = NULL;
-	xlator_t **     children = NULL;
-
-	int unwind     = 1;
-	int last_tried = -1;
-	int this_try = -1;
-	int read_child = -1;
-
-	priv     = this->private;
-	children = priv->children;
+    afr_local_t *local = NULL;
 
-	read_child = (long) cookie;
+    local = frame->local;
 
-	local = frame->local;
+    if (op_ret < 0) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
 
-	if (op_ret == -1) {
-	retry:
-		last_tried = local->cont.stat.last_tried;
-
-		if (all_tried (last_tried, priv->child_count)) {
-			goto out;
-		}
-		this_try = ++local->cont.stat.last_tried;
+        afr_read_txn_continue(frame, this, (long)cookie);
+        return 0;
+    }
 
-		if (this_try == read_child) {
-			goto retry;
-		}
+    AFR_STACK_UNWIND(fstat, frame, op_ret, op_errno, buf, xdata);
 
-		unwind = 0;
+    return 0;
+}
 
-		STACK_WIND_COOKIE (frame, afr_stat_cbk,
-				   (void *) (long) read_child,
-				   children[this_try],
-				   children[this_try]->fops->stat,
-				   &local->loc);
-	}
+int
+afr_fstat_wind(call_frame_t *frame, xlator_t *this, int subvol)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
 
-out:
-	if (unwind) {
-		if (buf)
-			buf->ia_ino = local->cont.stat.ino;
+    priv = this->private;
+    local = frame->local;
 
-		AFR_STACK_UNWIND (stat, frame, op_ret, op_errno, buf);
-	}
+    if (subvol == -1) {
+        AFR_STACK_UNWIND(fstat, frame, local->op_ret, local->op_errno, 0, 0);
+        return 0;
+    }
 
-	return 0;
+    STACK_WIND_COOKIE(
+        frame, afr_fstat_cbk, (void *)(long)subvol, priv->children[subvol],
+        priv->children[subvol]->fops->fstat, local->fd, local->xdata_req);
+    return 0;
 }
 
-
 int32_t
-afr_stat (call_frame_t *frame, xlator_t *this,
-	  loc_t *loc)
+afr_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-	afr_private_t * priv       = NULL;
-	afr_local_t   * local      = NULL;
-	xlator_t **     children   = NULL;
+    afr_local_t *local = NULL;
+    int op_errno = 0;
 
-        int32_t         read_child = -1;
-	int             call_child = 0;
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-	int32_t         op_ret     = -1;
-	int32_t         op_errno   = 0;
+    local->op = GF_FOP_FSTAT;
+    local->fd = fd_ref(fd);
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
 
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
+    afr_fix_open(fd, this);
 
-	priv     = this->private;
-	VALIDATE_OR_GOTO (priv->children, out);
+    afr_read_txn(frame, this, fd->inode, afr_fstat_wind, AFR_DATA_TRANSACTION);
 
-	children = priv->children;
+    return 0;
+out:
+    AFR_STACK_UNWIND(fstat, frame, -1, op_errno, NULL, NULL);
 
-	ALLOC_OR_GOTO (local, afr_local_t, out);
+    return 0;
+}
 
-	frame->local = local;
+/* }}} */
 
-        read_child = afr_read_child (this, loc->inode);
+/* {{{ readlink */
 
-        if ((read_child >= 0) && (priv->child_up[read_child])) {
-                call_child = read_child;
+int
+afr_readlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, const char *buf,
+                 struct iatt *sbuf, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
 
-                local->cont.stat.last_tried = -1;
+    local = frame->local;
 
-        } else {
-		call_child = afr_first_up_child (priv);
-		if (call_child == -1) {
-			op_errno = ENOTCONN;
-			gf_log (this->name, GF_LOG_DEBUG,
-				"no child is up");
-			goto out;
-		}
+    if (op_ret < 0) {
+        local->op_ret = -1;
+        local->op_errno = op_errno;
 
-		local->cont.stat.last_tried = call_child;
-	}
+        afr_read_txn_continue(frame, this, (long)cookie);
+        return 0;
+    }
 
-	loc_copy (&local->loc, loc);
+    AFR_STACK_UNWIND(readlink, frame, op_ret, op_errno, buf, sbuf, xdata);
+    return 0;
+}
 
-	local->cont.stat.ino = loc->inode->ino;
+int
+afr_readlink_wind(call_frame_t *frame, xlator_t *this, int subvol)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-	STACK_WIND_COOKIE (frame, afr_stat_cbk, (void *) (long) call_child,
-			   children[call_child],
-			   children[call_child]->fops->stat,
-			   loc);
+    local = frame->local;
+    priv = this->private;
 
-	op_ret = 0;
-out:
-	if (op_ret == -1) {
-		AFR_STACK_UNWIND (stat, frame, op_ret, op_errno, NULL);
-	}
+    if (subvol == -1) {
+        AFR_STACK_UNWIND(readlink, frame, local->op_ret, local->op_errno, 0, 0,
+                         0);
+        return 0;
+    }
 
-	return 0;
+    STACK_WIND_COOKIE(frame, afr_readlink_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->readlink, &local->loc,
+                      local->cont.readlink.size, local->xdata_req);
+    return 0;
 }
 
+int
+afr_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+             dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    int32_t op_errno = 0;
 
-/* }}} */
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-/* {{{ fstat */
+    local->op = GF_FOP_READLINK;
+    loc_copy(&local->loc, loc);
+    local->cont.readlink.size = size;
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
 
-int32_t
-afr_fstat_cbk (call_frame_t *frame, void *cookie,
-	       xlator_t *this, int32_t op_ret, int32_t op_errno,
-	       struct iatt *buf)
-{
-	afr_private_t * priv     = NULL;
-	afr_local_t *   local    = NULL;
-	xlator_t **     children = NULL;
+    afr_read_txn(frame, this, loc->inode, afr_readlink_wind,
+                 AFR_DATA_TRANSACTION);
+
+    return 0;
+out:
+    AFR_STACK_UNWIND(readlink, frame, -1, op_errno, 0, 0, 0);
+
+    return 0;
+}
 
-	int unwind     = 1;
-	int last_tried = -1;
-	int this_try = -1;
-        int read_child = -1;
+/* }}} */
 
-	priv     = this->private;
-	children = priv->children;
+/* {{{ getxattr */
 
-	local = frame->local;
+struct _xattr_key {
+    char *key;
+    struct list_head list;
+};
 
-	read_child = (long) cookie;
+int
+__gather_xattr_keys(dict_t *dict, char *key, data_t *value, void *data)
+{
+    struct list_head *list = data;
+    struct _xattr_key *xkey = NULL;
+
+    if (!strncmp(key, AFR_XATTR_PREFIX, SLEN(AFR_XATTR_PREFIX))) {
+        xkey = GF_MALLOC(sizeof(*xkey), gf_afr_mt_xattr_key);
+        if (!xkey)
+            return -1;
 
-	if (op_ret == -1) {
-	retry:
-		last_tried = local->cont.fstat.last_tried;
+        xkey->key = key;
+        INIT_LIST_HEAD(&xkey->list);
 
-		if (all_tried (last_tried, priv->child_count)) {
-			goto out;
-		}
-		this_try   = ++local->cont.fstat.last_tried;
+        list_add_tail(&xkey->list, list);
+    }
+    return 0;
+}
 
-		if (this_try == read_child) {
-			goto retry;
-		}
+void
+afr_filter_xattrs(dict_t *dict)
+{
+    struct list_head keys = {
+        0,
+    };
+    struct _xattr_key *key = NULL;
+    struct _xattr_key *tmp = NULL;
 
-		unwind = 0;
+    INIT_LIST_HEAD(&keys);
 
-		STACK_WIND_COOKIE (frame, afr_fstat_cbk,
-				   (void *) (long) read_child,
-				   children[this_try],
-				   children[this_try]->fops->fstat,
-				   local->fd);
-	}
+    dict_foreach(dict, __gather_xattr_keys, (void *)&keys);
 
-out:
-	if (unwind) {
-		if (buf)
-			buf->ia_ino = local->cont.fstat.ino;
+    list_for_each_entry_safe(key, tmp, &keys, list)
+    {
+        dict_del(dict, key->key);
 
-		AFR_STACK_UNWIND (fstat, frame, op_ret, op_errno, buf);
-	}
+        list_del_init(&key->list);
 
-	return 0;
+        GF_FREE(key);
+    }
 }
 
+static gf_boolean_t
+afr_getxattr_ignorable_errnos(int32_t op_errno)
+{
+    if (op_errno == ENODATA || op_errno == ENOTSUP || op_errno == ERANGE ||
+        op_errno == ENAMETOOLONG)
+        return _gf_true;
 
-int32_t
-afr_fstat (call_frame_t *frame, xlator_t *this,
-	   fd_t *fd)
+    return _gf_false;
+}
+int
+afr_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-	afr_private_t * priv       = NULL;
-	afr_local_t   * local      = NULL;
-	xlator_t **     children   = NULL;
+    afr_local_t *local = NULL;
 
-	int             call_child = 0;
-        int32_t         read_child = -1;
+    local = frame->local;
 
-	int32_t         op_ret     = -1;
-	int32_t         op_errno   = 0;
+    if (op_ret < 0 && !afr_getxattr_ignorable_errnos(op_errno)) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
 
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (fd, out);
-	VALIDATE_OR_GOTO (this->private, out);
+        afr_read_txn_continue(frame, this, (long)cookie);
+        return 0;
+    }
 
-	priv     = this->private;
-	VALIDATE_OR_GOTO (priv->children, out);
+    if (dict)
+        afr_filter_xattrs(dict);
 
-	children = priv->children;
+    AFR_STACK_UNWIND(getxattr, frame, op_ret, op_errno, dict, xdata);
 
-	ALLOC_OR_GOTO (local, afr_local_t, out);
+    return 0;
+}
 
-	frame->local = local;
+int
+afr_getxattr_wind(call_frame_t *frame, xlator_t *this, int subvol)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-	VALIDATE_OR_GOTO (fd->inode, out);
+    local = frame->local;
+    priv = this->private;
 
-        read_child = afr_read_child (this, fd->inode);
+    if (subvol == -1) {
+        AFR_STACK_UNWIND(getxattr, frame, local->op_ret, local->op_errno, NULL,
+                         NULL);
+        return 0;
+    }
 
-        if ((read_child >= 0) && (priv->child_up[read_child])) {
-                call_child = read_child;
+    STACK_WIND_COOKIE(frame, afr_getxattr_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->getxattr, &local->loc,
+                      local->cont.getxattr.name, local->xdata_req);
+    return 0;
+}
 
-                local->cont.fstat.last_tried = -1;
-        } else {
-                call_child = afr_first_up_child (priv);
+int32_t
+afr_getxattr_unwind(call_frame_t *frame, int op_ret, int op_errno, dict_t *dict,
+                    dict_t *xdata)
 
-                if (call_child == -1) {
-                        op_errno = ENOTCONN;
-                        gf_log (this->name, GF_LOG_DEBUG,
-				"no child is up");
-			goto out;
-		}
+{
+    AFR_STACK_UNWIND(getxattr, frame, op_ret, op_errno, dict, xdata);
+    return 0;
+}
 
-                local->cont.fstat.last_tried = call_child;
+int32_t
+afr_fgetxattr_clrlk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *dict,
+                        dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    xlator_t **children = NULL;
+    dict_t *xattr = NULL;
+    char *tmp_report = NULL;
+    char lk_summary[1024] = {
+        0,
+    };
+    int serz_len = 0;
+    int32_t callcnt = 0;
+    long int cky = 0;
+    int ret = 0;
+    int keylen = 0;
+    int children_keylen = 0;
+
+    priv = this->private;
+    children = priv->children;
+
+    local = frame->local;
+    cky = (long)cookie;
+    keylen = strlen(local->cont.getxattr.name);
+    children_keylen = strlen(children[cky]->name);
+
+    LOCK(&frame->lock);
+    {
+        callcnt = --local->call_count;
+        if (op_ret == -1)
+            local->replies[cky].op_errno = op_errno;
+
+        if (!local->dict)
+            local->dict = dict_new();
+        if (local->dict) {
+            ret = dict_get_strn(dict, local->cont.getxattr.name, keylen,
+                                &tmp_report);
+            if (ret)
+                goto unlock;
+            ret = dict_set_dynstrn(local->dict, children[cky]->name,
+                                   children_keylen, gf_strdup(tmp_report));
+            if (ret)
+                goto unlock;
+        }
+    }
+unlock:
+    UNLOCK(&frame->lock);
+
+    if (!callcnt) {
+        xattr = dict_new();
+        if (!xattr) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            goto unwind;
+        }
+        ret = dict_serialize_value_with_delim(local->dict, lk_summary,
+                                              &serz_len, '\n');
+        if (ret) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            goto unwind;
+        }
+        if (serz_len == -1)
+            snprintf(lk_summary, sizeof(lk_summary), "No locks cleared.");
+        ret = dict_set_dynstrn(xattr, local->cont.getxattr.name, keylen,
+                               gf_strdup(lk_summary));
+        if (ret) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_DICT_SET_FAILED,
+                   "Error setting dictionary");
+            goto unwind;
         }
 
-	local->cont.fstat.ino = fd->inode->ino;
-	local->fd = fd_ref (fd);
-
-	STACK_WIND_COOKIE (frame, afr_fstat_cbk, (void *) (long) call_child,
-			   children[call_child],
-			   children[call_child]->fops->fstat,
-			   fd);
+        op_errno = afr_final_errno(local, priv);
 
-	op_ret = 0;
-out:
-	if (op_ret == -1) {
-		AFR_STACK_UNWIND (fstat, frame, op_ret, op_errno, NULL);
-	}
+    unwind:
+        AFR_STACK_UNWIND(fgetxattr, frame, op_ret, op_errno, xattr, xdata);
+        if (xattr)
+            dict_unref(xattr);
+    }
 
-	return 0;
+    return ret;
 }
 
-/* }}} */
-
-/* {{{ readlink */
-
 int32_t
-afr_readlink_cbk (call_frame_t *frame, void *cookie,
-		  xlator_t *this, int32_t op_ret, int32_t op_errno,
-		  const char *buf, struct iatt *sbuf)
+afr_getxattr_clrlk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *dict,
+                       dict_t *xdata)
 {
-	afr_private_t * priv     = NULL;
-	afr_local_t *   local    = NULL;
-	xlator_t **     children = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    xlator_t **children = NULL;
+    dict_t *xattr = NULL;
+    char *tmp_report = NULL;
+    char lk_summary[1024] = {
+        0,
+    };
+    int serz_len = 0;
+    int32_t callcnt = 0;
+    long int cky = 0;
+    int ret = 0;
+    int keylen = 0;
+    int children_keylen = 0;
+
+    priv = this->private;
+    children = priv->children;
+
+    local = frame->local;
+    cky = (long)cookie;
+
+    keylen = strlen(local->cont.getxattr.name);
+    children_keylen = strlen(children[cky]->name);
+
+    LOCK(&frame->lock);
+    {
+        callcnt = --local->call_count;
+        if (op_ret == -1)
+            local->replies[cky].op_errno = op_errno;
+
+        if (!local->dict)
+            local->dict = dict_new();
+        if (local->dict) {
+            ret = dict_get_strn(dict, local->cont.getxattr.name, keylen,
+                                &tmp_report);
+            if (ret)
+                goto unlock;
+            ret = dict_set_dynstrn(local->dict, children[cky]->name,
+                                   children_keylen, gf_strdup(tmp_report));
+            if (ret)
+                goto unlock;
+        }
+    }
+unlock:
+    UNLOCK(&frame->lock);
+
+    if (!callcnt) {
+        xattr = dict_new();
+        if (!xattr) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            goto unwind;
+        }
+        ret = dict_serialize_value_with_delim(local->dict, lk_summary,
+                                              &serz_len, '\n');
+        if (ret) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            goto unwind;
+        }
+        if (serz_len == -1)
+            snprintf(lk_summary, sizeof(lk_summary), "No locks cleared.");
+        ret = dict_set_dynstrn(xattr, local->cont.getxattr.name, keylen,
+                               gf_strdup(lk_summary));
+        if (ret) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_DICT_SET_FAILED,
+                   "Error setting dictionary");
+            goto unwind;
+        }
 
-	int unwind     = 1;
-	int last_tried = -1;
-	int this_try = -1;
-        int read_child = -1;
+        op_errno = afr_final_errno(local, priv);
 
-	priv     = this->private;
-	children = priv->children;
+    unwind:
+        AFR_STACK_UNWIND(getxattr, frame, op_ret, op_errno, xattr, xdata);
 
-	local = frame->local;
+        if (xattr)
+            dict_unref(xattr);
+    }
 
-        read_child = (long) cookie;
+    return ret;
+}
 
-	if (op_ret == -1) {
-        retry:
-		last_tried = local->cont.readlink.last_tried;
+/**
+ * node-uuid cbk uses next child querying mechanism
+ */
+int32_t
+afr_getxattr_node_uuid_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, dict_t *dict,
+                           dict_t *xdata)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    xlator_t **children = NULL;
+    int unwind = 1;
+    int curr_call_child = 0;
+
+    priv = this->private;
+    children = priv->children;
+
+    local = frame->local;
+
+    if (op_ret == -1) { /** query the _next_ child */
+
+        /**
+         * _current_ becomes _next_
+         * If done with all children and yet no success; give up !
+         */
+        curr_call_child = (int)((long)cookie);
+        if (++curr_call_child == priv->child_count)
+            goto unwind;
+
+        gf_msg_debug(this->name, op_errno,
+                     "op_ret (-1): Re-querying afr-child (%d/%d)",
+                     curr_call_child, priv->child_count);
+
+        unwind = 0;
+        STACK_WIND_COOKIE(
+            frame, afr_getxattr_node_uuid_cbk, (void *)(long)curr_call_child,
+            children[curr_call_child],
+            children[curr_call_child]->fops->getxattr, &local->loc,
+            local->cont.getxattr.name, local->xdata_req);
+    }
+
+unwind:
+    if (unwind)
+        AFR_STACK_UNWIND(getxattr, frame, op_ret, op_errno, dict, xdata);
+
+    return 0;
+}
 
-		if (all_tried (last_tried, priv->child_count)) {
-			goto out;
-		}
-		this_try = ++local->cont.readlink.last_tried;
+/**
+ * list-node-uuids cbk returns the list of node_uuids for the subvolume.
+ */
+int32_t
+afr_getxattr_list_node_uuids_cbk(call_frame_t *frame, void *cookie,
+                                 xlator_t *this, int32_t op_ret,
+                                 int32_t op_errno, dict_t *dict, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int32_t callcnt = 0;
+    int ret = 0;
+    char *xattr_serz = NULL;
+    long cky = 0;
+    int32_t tlen = 0;
+
+    local = frame->local;
+    priv = this->private;
+    cky = (long)cookie;
+
+    LOCK(&frame->lock);
+    {
+        callcnt = --local->call_count;
+        local->replies[cky].valid = 1;
+        local->replies[cky].op_ret = op_ret;
+        local->replies[cky].op_errno = op_errno;
+
+        if (op_ret < 0)
+            goto unlock;
+
+        local->op_ret = 0;
+
+        if (!local->xdata_rsp && xdata)
+            local->xdata_rsp = dict_ref(xdata);
+        local->replies[cky].xattr = dict_ref(dict);
+    }
+
+unlock:
+    UNLOCK(&frame->lock);
+
+    if (!callcnt) {
+        if (local->op_ret != 0) {
+            /* All bricks gave an error. */
+            local->op_errno = afr_final_errno(local, priv);
+            goto unwind;
+        }
 
-                if (this_try == read_child) {
-                        goto retry;
-                }
+        /*Since we store the UUID0_STR as node uuid for down bricks and
+         *for non zero op_ret, assigning length to  priv->child_count
+         *number of uuids*/
+        local->cont.getxattr.xattr_len = (SLEN(UUID0_STR) + 2) *
+                                         priv->child_count;
+
+        if (!local->dict)
+            local->dict = dict_new();
+        if (!local->dict) {
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+            goto unwind;
+        }
 
-		unwind = 0;
-		STACK_WIND_COOKIE (frame, afr_readlink_cbk,
-				   (void *) (long) read_child,
-				   children[this_try],
-				   children[this_try]->fops->readlink,
-				   &local->loc,
-				   local->cont.readlink.size);
-	}
+        xattr_serz = GF_CALLOC(local->cont.getxattr.xattr_len, sizeof(char),
+                               gf_common_mt_char);
 
-out:
-	if (unwind) {
-                if (sbuf)
-                        sbuf->ia_ino = local->cont.readlink.ino;
+        if (!xattr_serz) {
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+            goto unwind;
+        }
 
-		AFR_STACK_UNWIND (readlink, frame, op_ret, op_errno, buf, sbuf);
-	}
+        ret = afr_serialize_xattrs_with_delimiter(frame, this, xattr_serz,
+                                                  UUID0_STR, &tlen, ' ');
+        if (ret) {
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+            GF_FREE(xattr_serz);
+            goto unwind;
+        }
+        ret = dict_set_dynstr_sizen(local->dict, GF_XATTR_LIST_NODE_UUIDS_KEY,
+                                    xattr_serz);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+                   "Cannot set node_uuid key in dict");
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+            if (ret == -EINVAL)
+                GF_FREE(xattr_serz);
+        } else {
+            local->op_ret = local->cont.getxattr.xattr_len - 1;
+            local->op_errno = 0;
+        }
 
-	return 0;
+    unwind:
+        AFR_STACK_UNWIND(getxattr, frame, local->op_ret, local->op_errno,
+                         local->dict, local->xdata_rsp);
+    }
+
+    return ret;
 }
 
+int32_t
+afr_getxattr_quota_size_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno, dict_t *dict,
+                            dict_t *xdata)
+{
+    int idx = (long)cookie;
+    int call_count = 0;
+    afr_local_t *local = frame->local;
+    int read_subvol = -1;
+
+    local->replies[idx].valid = 1;
+    local->replies[idx].op_ret = op_ret;
+    local->replies[idx].op_errno = op_errno;
+    if (dict)
+        local->replies[idx].xdata = dict_ref(dict);
+    call_count = afr_frame_return(frame);
+    if (call_count == 0) {
+        local->inode = inode_ref(local->loc.inode);
+        read_subvol = afr_handle_quota_size(frame, this);
+        if (read_subvol != -1) {
+            op_ret = local->replies[read_subvol].op_ret;
+            op_errno = local->replies[read_subvol].op_errno;
+            dict = local->replies[read_subvol].xdata;
+        }
+        AFR_STACK_UNWIND(getxattr, frame, op_ret, op_errno, dict, xdata);
+    }
+
+    return 0;
+}
 
 int32_t
-afr_readlink (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, size_t size)
+afr_getxattr_lockinfo_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, dict_t *dict,
+                          dict_t *xdata)
 {
-	afr_private_t * priv       = NULL;
-	xlator_t **     children   = NULL;
-	int             call_child = 0;
-	afr_local_t     *local     = NULL;
+    int call_cnt = 0, len = 0;
+    char *lockinfo_buf = NULL;
+    dict_t *lockinfo = NULL, *newdict = NULL;
+    afr_local_t *local = NULL;
 
-        int32_t         read_child = -1;
+    LOCK(&frame->lock);
+    {
+        local = frame->local;
 
-	int32_t op_ret   = -1;
-	int32_t op_errno = 0;
+        call_cnt = --local->call_count;
 
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
+        if ((op_ret < 0) || (!dict && !xdata)) {
+            goto unlock;
+        }
 
-	priv     = this->private;
-	VALIDATE_OR_GOTO (priv->children, out);
+        if (xdata) {
+            if (!local->xdata_rsp) {
+                local->xdata_rsp = dict_new();
+                if (!local->xdata_rsp) {
+                    local->op_ret = -1;
+                    local->op_errno = ENOMEM;
+                    goto unlock;
+                }
+            }
+        }
 
-	children = priv->children;
+        if (!dict) {
+            goto unlock;
+        }
 
-	ALLOC_OR_GOTO (local, afr_local_t, out);
+        op_ret = dict_get_ptr_and_len(dict, GF_XATTR_LOCKINFO_KEY,
+                                      (void **)&lockinfo_buf, &len);
 
-	frame->local = local;
+        if (!lockinfo_buf) {
+            goto unlock;
+        }
 
-        read_child = afr_read_child (this, loc->inode);
+        if (!local->dict) {
+            local->dict = dict_new();
+            if (!local->dict) {
+                local->op_ret = -1;
+                local->op_errno = ENOMEM;
+                goto unlock;
+            }
+        }
+    }
+unlock:
+    UNLOCK(&frame->lock);
+
+    if (lockinfo_buf != NULL) {
+        lockinfo = dict_new();
+        if (lockinfo == NULL) {
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+        } else {
+            op_ret = dict_unserialize(lockinfo_buf, len, &lockinfo);
 
-        if ((read_child >= 0) && (priv->child_up[read_child])) {
-                call_child = read_child;
+            if (lockinfo && local->dict) {
+                dict_copy(lockinfo, local->dict);
+            }
+        }
+    }
+
+    if (xdata && local->xdata_rsp) {
+        dict_copy(xdata, local->xdata_rsp);
+    }
+
+    if (!call_cnt) {
+        newdict = dict_new();
+        if (!newdict) {
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+            goto unwind;
+        }
 
-                local->cont.readlink.last_tried = -1;
+        op_ret = dict_allocate_and_serialize(
+            local->dict, (char **)&lockinfo_buf, (unsigned int *)&len);
+        if (op_ret != 0) {
+            local->op_ret = -1;
+            goto unwind;
+        }
 
-        } else {
-                call_child = afr_first_up_child (priv);
+        op_ret = dict_set_dynptr(newdict, GF_XATTR_LOCKINFO_KEY,
+                                 (void *)lockinfo_buf, len);
+        if (op_ret < 0) {
+            local->op_ret = -1;
+            local->op_errno = -op_ret;
+            goto unwind;
+        }
+
+    unwind:
+        AFR_STACK_UNWIND(getxattr, frame, op_ret, op_errno, newdict,
+                         local->xdata_rsp);
+    }
+
+    dict_unref(lockinfo);
+
+    return 0;
+}
+
+int32_t
+afr_fgetxattr_lockinfo_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, dict_t *dict,
+                           dict_t *xdata)
+{
+    int call_cnt = 0, len = 0;
+    char *lockinfo_buf = NULL;
+    dict_t *lockinfo = NULL, *newdict = NULL;
+    afr_local_t *local = NULL;
+
+    LOCK(&frame->lock);
+    {
+        local = frame->local;
 
-                if (call_child == -1) {
-                        op_errno = ENOTCONN;
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "no child is up");
-                        goto out;
+        call_cnt = --local->call_count;
+
+        if ((op_ret < 0) || (!dict && !xdata)) {
+            goto unlock;
+        }
+
+        if (xdata) {
+            if (!local->xdata_rsp) {
+                local->xdata_rsp = dict_new();
+                if (!local->xdata_rsp) {
+                    local->op_ret = -1;
+                    local->op_errno = ENOMEM;
+                    goto unlock;
                 }
+            }
+        }
 
-                local->cont.readlink.last_tried = call_child;
+        if (!dict) {
+            goto unlock;
         }
 
-	loc_copy (&local->loc, loc);
+        op_ret = dict_get_ptr_and_len(dict, GF_XATTR_LOCKINFO_KEY,
+                                      (void **)&lockinfo_buf, &len);
 
-	local->cont.readlink.size       = size;
-        local->cont.readlink.ino        = loc->inode->ino;
+        if (!lockinfo_buf) {
+            goto unlock;
+        }
 
-	STACK_WIND_COOKIE (frame, afr_readlink_cbk,
-			   (void *) (long) call_child,
-			   children[call_child], children[call_child]->fops->readlink,
-			   loc, size);
+        if (!local->dict) {
+            local->dict = dict_new();
+            if (!local->dict) {
+                local->op_ret = -1;
+                local->op_errno = ENOMEM;
+                goto unlock;
+            }
+        }
+    }
+unlock:
+    UNLOCK(&frame->lock);
+
+    if (lockinfo_buf != NULL) {
+        lockinfo = dict_new();
+        if (lockinfo == NULL) {
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+        } else {
+            op_ret = dict_unserialize(lockinfo_buf, len, &lockinfo);
 
-	op_ret = 0;
-out:
-	if (op_ret == -1) {
-		AFR_STACK_UNWIND (readlink, frame, op_ret, op_errno, NULL, NULL);
-	}
-	return 0;
-}
+            if (lockinfo && local->dict) {
+                dict_copy(lockinfo, local->dict);
+            }
+        }
+    }
+
+    if (xdata && local->xdata_rsp) {
+        dict_copy(xdata, local->xdata_rsp);
+    }
+
+    if (!call_cnt) {
+        newdict = dict_new();
+        if (!newdict) {
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+            goto unwind;
+        }
 
+        op_ret = dict_allocate_and_serialize(
+            local->dict, (char **)&lockinfo_buf, (unsigned int *)&len);
+        if (op_ret != 0) {
+            local->op_ret = -1;
+            goto unwind;
+        }
 
-/* }}} */
+        op_ret = dict_set_dynptr(newdict, GF_XATTR_LOCKINFO_KEY,
+                                 (void *)lockinfo_buf, len);
+        if (op_ret < 0) {
+            local->op_ret = -1;
+            local->op_errno = -op_ret;
+            goto unwind;
+        }
 
-/* {{{ getxattr */
+    unwind:
+        AFR_STACK_UNWIND(fgetxattr, frame, op_ret, op_errno, newdict,
+                         local->xdata_rsp);
+    }
 
-struct _xattr_key {
-        char *key;
-        struct list_head list;
-};
+    dict_unref(lockinfo);
 
+    return 0;
+}
 
-void
-__gather_xattr_keys (dict_t *dict, char *key, data_t *value,
-                     void *data)
+int32_t
+afr_fgetxattr_pathinfo_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, dict_t *dict,
+                           dict_t *xdata)
 {
-        struct list_head *  list  = data;
-        struct _xattr_key * xkey  = NULL;
+    afr_local_t *local = NULL;
+    int32_t callcnt = 0;
+    int ret = 0;
+    char *xattr = NULL;
+    char *xattr_serz = NULL;
+    int keylen = 0;
+    char xattr_cky[1024] = {
+        0,
+    };
+    int xattr_cky_len = 0;
+    dict_t *nxattr = NULL;
+    long cky = 0;
+    int32_t padding = 0;
+    int32_t tlen = 0;
+
+    if (!frame || !frame->local || !this) {
+        gf_msg("", GF_LOG_ERROR, 0, AFR_MSG_INVALID_ARG, "possible NULL deref");
+        goto out;
+    }
+
+    local = frame->local;
+    cky = (long)cookie;
+    keylen = strlen(local->cont.getxattr.name);
+    xattr_cky_len = snprintf(xattr_cky, sizeof(xattr_cky), "%s-%ld",
+                             local->cont.getxattr.name, cky);
+    LOCK(&frame->lock);
+    {
+        callcnt = --local->call_count;
+
+        if (op_ret < 0) {
+            local->op_errno = op_errno;
+        } else {
+            local->op_ret = op_ret;
+            if (!local->xdata_rsp && xdata)
+                local->xdata_rsp = dict_ref(xdata);
+        }
 
-        if (!strncmp (key, AFR_XATTR_PREFIX,
-                      strlen (AFR_XATTR_PREFIX))) {
+        if (!dict || (op_ret < 0))
+            goto unlock;
 
-                xkey = GF_CALLOC (1, sizeof (*xkey), gf_afr_mt_xattr_key);
-                if (!xkey)
-                        return;
+        if (!local->dict) {
+            local->dict = dict_new();
+            if (!local->dict)
+                goto unlock;
+        }
+        ret = dict_get_strn(dict, local->cont.getxattr.name, keylen, &xattr);
+        if (ret)
+            goto unlock;
+
+        xattr = gf_strdup(xattr);
+
+        ret = dict_set_dynstrn(local->dict, xattr_cky, xattr_cky_len, xattr);
+        if (ret) {
+            UNLOCK(&frame->lock);
+            gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+                   "Cannot set xattr cookie key");
+            goto post_unlock;
+        }
 
-                xkey->key = key;
-                INIT_LIST_HEAD (&xkey->list);
+        local->cont.getxattr.xattr_len += strlen(xattr) + 1;
+    }
+unlock:
+    UNLOCK(&frame->lock);
+post_unlock:
+    if (!callcnt) {
+        if (!local->cont.getxattr.xattr_len)
+            goto unwind;
+
+        nxattr = dict_new();
+        if (!nxattr)
+            goto unwind;
+
+        /* extra bytes for decorations (brackets and <>'s) */
+        padding += strlen(this->name) + SLEN(AFR_PATHINFO_HEADER) + 4;
+        local->cont.getxattr.xattr_len += (padding + 2);
+
+        xattr_serz = GF_MALLOC(local->cont.getxattr.xattr_len,
+                               gf_common_mt_char);
+
+        if (!xattr_serz)
+            goto unwind;
+
+        /* the xlator info */
+        int xattr_serz_len = sprintf(
+            xattr_serz, "(<" AFR_PATHINFO_HEADER "%s> ", this->name);
+
+        /* actual series of pathinfo */
+        ret = dict_serialize_value_with_delim(
+            local->dict, xattr_serz + xattr_serz_len, &tlen, ' ');
+        if (ret) {
+            GF_FREE(xattr_serz);
+            goto unwind;
+        }
 
-                list_add_tail (&xkey->list, list);
+        /* closing part */
+        *(xattr_serz + padding + tlen) = ')';
+        *(xattr_serz + padding + tlen + 1) = '\0';
+
+        ret = dict_set_dynstrn(nxattr, local->cont.getxattr.name, keylen,
+                               xattr_serz);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+                   "Cannot set pathinfo key in dict");
+            if (ret == -EINVAL)
+                GF_FREE(xattr_serz);
         }
-}
 
+    unwind:
+        AFR_STACK_UNWIND(fgetxattr, frame, local->op_ret, local->op_errno,
+                         nxattr, local->xdata_rsp);
 
-void
-__filter_xattrs (dict_t *dict)
-{
-        struct list_head keys;
+        if (nxattr)
+            dict_unref(nxattr);
+    }
 
-        struct _xattr_key *key;
-        struct _xattr_key *tmp;
+out:
+    return ret;
+}
 
-        INIT_LIST_HEAD (&keys);
+int32_t
+afr_getxattr_pathinfo_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, dict_t *dict,
+                          dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    int32_t callcnt = 0;
+    int ret = 0;
+    char *xattr = NULL;
+    char *xattr_serz = NULL;
+    char xattr_cky[1024] = {
+        0,
+    };
+    int keylen = 0;
+    int xattr_cky_len = 0;
+    dict_t *nxattr = NULL;
+    long cky = 0;
+    int32_t padding = 0;
+    int32_t tlen = 0;
+
+    if (!frame || !frame->local || !this) {
+        gf_msg("", GF_LOG_ERROR, 0, AFR_MSG_INVALID_ARG, "possible NULL deref");
+        goto out;
+    }
+
+    local = frame->local;
+    cky = (long)cookie;
+    keylen = strlen(local->cont.getxattr.name);
+    xattr_cky_len = snprintf(xattr_cky, sizeof(xattr_cky), "%s-%ld",
+                             local->cont.getxattr.name, cky);
+    LOCK(&frame->lock);
+    {
+        callcnt = --local->call_count;
+
+        if (op_ret < 0) {
+            local->op_errno = op_errno;
+        } else {
+            local->op_ret = op_ret;
+            if (!local->xdata_rsp && xdata)
+                local->xdata_rsp = dict_ref(xdata);
+        }
 
-        dict_foreach (dict, __gather_xattr_keys,
-                      (void *) &keys);
+        if (!dict || (op_ret < 0))
+            goto unlock;
 
-        list_for_each_entry_safe (key, tmp, &keys, list) {
-                dict_del (dict, key->key);
+        if (!local->dict) {
+            local->dict = dict_new();
+            if (!local->dict)
+                goto unlock;
+        }
+        ret = dict_get_strn(dict, local->cont.getxattr.name, keylen, &xattr);
+        if (ret)
+            goto unlock;
+
+        xattr = gf_strdup(xattr);
+
+        ret = dict_set_dynstrn(local->dict, xattr_cky, xattr_cky_len, xattr);
+        if (ret) {
+            UNLOCK(&frame->lock);
+            gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+                   "Cannot set xattr cookie key");
+            goto post_unlock;
+        }
 
-                list_del_init (&key->list);
+        local->cont.getxattr.xattr_len += strlen(xattr) + 1;
+    }
+unlock:
+    UNLOCK(&frame->lock);
+post_unlock:
+    if (!callcnt) {
+        if (!local->cont.getxattr.xattr_len)
+            goto unwind;
+
+        nxattr = dict_new();
+        if (!nxattr)
+            goto unwind;
+
+        /* extra bytes for decorations (brackets and <>'s) */
+        padding += strlen(this->name) + SLEN(AFR_PATHINFO_HEADER) + 4;
+        local->cont.getxattr.xattr_len += (padding + 2);
+
+        xattr_serz = GF_MALLOC(local->cont.getxattr.xattr_len,
+                               gf_common_mt_char);
+
+        if (!xattr_serz)
+            goto unwind;
+
+        /* the xlator info */
+        int xattr_serz_len = sprintf(
+            xattr_serz, "(<" AFR_PATHINFO_HEADER "%s> ", this->name);
+
+        /* actual series of pathinfo */
+        ret = dict_serialize_value_with_delim(
+            local->dict, xattr_serz + xattr_serz_len, &tlen, ' ');
+        if (ret) {
+            GF_FREE(xattr_serz);
+            goto unwind;
+        }
 
-                GF_FREE (key);
+        /* closing part */
+        *(xattr_serz + padding + tlen) = ')';
+        *(xattr_serz + padding + tlen + 1) = '\0';
+
+        ret = dict_set_dynstrn(nxattr, local->cont.getxattr.name, keylen,
+                               xattr_serz);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+                   "Cannot set pathinfo key in dict");
+            if (ret == -EINVAL)
+                GF_FREE(xattr_serz);
         }
+
+    unwind:
+        AFR_STACK_UNWIND(getxattr, frame, local->op_ret, local->op_errno,
+                         nxattr, local->xdata_rsp);
+
+        if (nxattr)
+            dict_unref(nxattr);
+    }
+
+out:
+    return ret;
 }
 
+static int
+afr_aggregate_stime_xattr(dict_t *this, char *key, data_t *value, void *data)
+{
+    int ret = 0;
+
+    if (fnmatch(GF_XATTR_STIME_PATTERN, key, FNM_NOESCAPE) == 0)
+        ret = gf_get_max_stime(THIS, data, key, value);
 
+    return ret;
+}
 
 int32_t
-afr_getxattr_cbk (call_frame_t *frame, void *cookie,
-		  xlator_t *this, int32_t op_ret, int32_t op_errno,
-		  dict_t *dict)
+afr_common_getxattr_stime_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                              int32_t op_ret, int32_t op_errno, dict_t *dict,
+                              dict_t *xdata)
 {
-	afr_private_t * priv     = NULL;
-	afr_local_t *   local    = NULL;
-	xlator_t **     children = NULL;
+    afr_local_t *local = NULL;
+    int32_t callcnt = 0;
 
-	int unwind     = 1;
-	int last_tried = -1;
-	int this_try = -1;
-        int read_child = -1;
+    if (!frame || !frame->local || !this) {
+        gf_msg("", GF_LOG_ERROR, 0, AFR_MSG_INVALID_ARG, "possible NULL deref");
+        goto out;
+    }
 
-	priv     = this->private;
-	children = priv->children;
+    local = frame->local;
 
-	local = frame->local;
+    LOCK(&frame->lock);
+    {
+        callcnt = --local->call_count;
 
-        read_child = (long) cookie;
+        if (!dict || (op_ret < 0)) {
+            local->op_errno = op_errno;
+            goto cleanup;
+        }
 
-	if (op_ret == -1) {
-        retry:
-		last_tried = local->cont.getxattr.last_tried;
+        if (!local->dict)
+            local->dict = dict_copy_with_ref(dict, NULL);
+        else
+            dict_foreach(dict, afr_aggregate_stime_xattr, local->dict);
+        local->op_ret = 0;
+    }
 
-		if (all_tried (last_tried, priv->child_count)) {
-			goto out;
-		}
-		this_try = ++local->cont.getxattr.last_tried;
+cleanup:
+    UNLOCK(&frame->lock);
 
-                if (this_try == read_child) {
-                        goto retry;
-                }
-
-		unwind = 0;
-		STACK_WIND_COOKIE (frame, afr_getxattr_cbk,
-				   (void *) (long) read_child,
-				   children[this_try],
-				   children[this_try]->fops->getxattr,
-				   &local->loc,
-				   local->cont.getxattr.name);
-	}
+    if (!callcnt) {
+        AFR_STACK_UNWIND(getxattr, frame, local->op_ret, local->op_errno,
+                         local->dict, xdata);
+    }
 
 out:
-	if (unwind) {
-                if (op_ret >= 0 && dict)
-                        __filter_xattrs (dict);
+    return 0;
+}
 
-		AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict);
-	}
+static gf_boolean_t
+afr_is_special_xattr(const char *name, fop_getxattr_cbk_t *cbk,
+                     gf_boolean_t is_fgetxattr)
+{
+    gf_boolean_t is_spl = _gf_true;
+
+    GF_ASSERT(cbk);
+    if (!cbk || !name) {
+        is_spl = _gf_false;
+        goto out;
+    }
+
+    if (!strcmp(name, GF_XATTR_PATHINFO_KEY) ||
+        !strcmp(name, GF_XATTR_USER_PATHINFO_KEY)) {
+        if (is_fgetxattr) {
+            *cbk = afr_fgetxattr_pathinfo_cbk;
+        } else {
+            *cbk = afr_getxattr_pathinfo_cbk;
+        }
+    } else if (!strncmp(name, GF_XATTR_CLRLK_CMD, SLEN(GF_XATTR_CLRLK_CMD))) {
+        if (is_fgetxattr) {
+            *cbk = afr_fgetxattr_clrlk_cbk;
+        } else {
+            *cbk = afr_getxattr_clrlk_cbk;
+        }
+    } else if (!strncmp(name, GF_XATTR_LOCKINFO_KEY,
+                        SLEN(GF_XATTR_LOCKINFO_KEY))) {
+        if (is_fgetxattr) {
+            *cbk = afr_fgetxattr_lockinfo_cbk;
+        } else {
+            *cbk = afr_getxattr_lockinfo_cbk;
+        }
+    } else if (fnmatch(GF_XATTR_STIME_PATTERN, name, FNM_NOESCAPE) == 0) {
+        *cbk = afr_common_getxattr_stime_cbk;
+    } else if (strcmp(name, QUOTA_SIZE_KEY) == 0) {
+        *cbk = afr_getxattr_quota_size_cbk;
+    } else if (!strcmp(name, GF_XATTR_LIST_NODE_UUIDS_KEY)) {
+        *cbk = afr_getxattr_list_node_uuids_cbk;
+    } else {
+        is_spl = _gf_false;
+    }
 
-	return 0;
+out:
+    return is_spl;
 }
 
-int32_t
-afr_getxattr_unwind (void *arg, call_frame_t *frame,
-                        int op_ret, int op_errno, dict_t *dict)
+static void
+afr_getxattr_all_subvols(xlator_t *this, call_frame_t *frame, const char *name,
+                         loc_t *loc, fop_getxattr_cbk_t cbk)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int i = 0;
+    int call_count = 0;
+
+    priv = this->private;
+
+    local = frame->local;
+    // local->call_count set in afr_local_init
+    call_count = local->call_count;
+
+    if (!strcmp(name, GF_XATTR_LIST_NODE_UUIDS_KEY)) {
+        GF_FREE(local->cont.getxattr.name);
+        local->cont.getxattr.name = gf_strdup(GF_XATTR_NODE_UUID_KEY);
+    }
+
+    // If up-children count is 0, afr_local_init would have failed already
+    // and the call would have unwound so not handling it here.
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->child_up[i]) {
+            STACK_WIND_COOKIE(frame, cbk, (void *)(long)i, priv->children[i],
+                              priv->children[i]->fops->getxattr, loc,
+                              local->cont.getxattr.name, NULL);
+            if (!--call_count)
+                break;
+        }
+    }
+    return;
+}
 
+int
+afr_marker_populate_args(call_frame_t *frame, int type, int *gauge,
+                         xlator_t **subvols)
 {
-        AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict);
-        return 0;
+    xlator_t *this = frame->this;
+    afr_private_t *priv = this->private;
+
+    memcpy(subvols, priv->children, sizeof(*subvols) * priv->child_count);
+
+    if (type == MARKER_XTIME_TYPE) {
+        /*Don't error out on ENOENT/ENOTCONN */
+        gauge[MCNT_NOTFOUND] = 0;
+        gauge[MCNT_ENOTCONN] = 0;
+    }
+    return priv->child_count;
+}
+
+static int
+afr_handle_heal_xattrs(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       const char *heal_op)
+{
+    int ret = -1;
+    afr_spb_status_t *data = NULL;
+
+    if (!strcmp(heal_op, GF_HEAL_INFO)) {
+        afr_get_heal_info(frame, this, loc);
+        ret = 0;
+        goto out;
+    }
+
+    if (!strcmp(heal_op, GF_AFR_HEAL_SBRAIN)) {
+        afr_heal_splitbrain_file(frame, this, loc);
+        ret = 0;
+        goto out;
+    }
+
+    if (!strcmp(heal_op, GF_AFR_SBRAIN_STATUS)) {
+        data = GF_CALLOC(1, sizeof(*data), gf_afr_mt_spb_status_t);
+        if (!data) {
+            ret = 1;
+            goto out;
+        }
+        data->frame = frame;
+        data->loc = loc;
+        ret = synctask_new(this->ctx->env, afr_get_split_brain_status,
+                           afr_get_split_brain_status_cbk, NULL, data);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN_STATUS,
+                   "Failed to create"
+                   " synctask. Unable to fetch split-brain status"
+                   " for %s.",
+                   loc->name);
+            ret = 1;
+            goto out;
+        }
+        goto out;
+    }
+
+out:
+    if (ret == 1) {
+        AFR_STACK_UNWIND(getxattr, frame, -1, ENOMEM, NULL, NULL);
+        if (data)
+            GF_FREE(data);
+        ret = 0;
+    }
+    return ret;
 }
 
 int32_t
-afr_getxattr (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, const char *name)
+afr_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, const char *name,
+             dict_t *xdata)
 {
-	afr_private_t *   priv       = NULL;
-	xlator_t **       children   = NULL;
-	int               call_child = 0;
-	afr_local_t     * local      = NULL;
-        xlator_list_t   * trav       = NULL;
-        xlator_t       ** sub_volumes= NULL;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    xlator_t **children = NULL;
+    int i = 0;
+    int32_t op_errno = 0;
+    int ret = -1;
+    fop_getxattr_cbk_t cbk = NULL;
 
-        int               read_child = -1;
-        int               i          = 0;
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-	int32_t op_ret   = -1;
-	int32_t op_errno = 0;
+    priv = this->private;
 
+    children = priv->children;
 
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
+    loc_copy(&local->loc, loc);
 
-	priv     = this->private;
-	VALIDATE_OR_GOTO (priv->children, out);
+    local->op = GF_FOP_GETXATTR;
 
-	children = priv->children;
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
 
-	ALLOC_OR_GOTO (local, afr_local_t, out);
-	frame->local = local;
+    if (!name)
+        goto no_name;
 
-        loc_copy (&local->loc, loc);
-        if (name)
-                local->cont.getxattr.name       = gf_strdup (name);
+    local->cont.getxattr.name = gf_strdup(name);
 
+    if (!local->cont.getxattr.name) {
+        op_errno = ENOMEM;
+        goto out;
+    }
 
-        if (name) {
-                if (!strncmp (name, AFR_XATTR_PREFIX,
-                              strlen (AFR_XATTR_PREFIX))) {
+    if (!strncmp(name, AFR_XATTR_PREFIX, SLEN(AFR_XATTR_PREFIX))) {
+        op_errno = ENODATA;
+        goto out;
+    }
 
-                        op_errno = ENODATA;
-                        goto out;
-                }
-                if ((strcmp (GF_XATTR_MARKER_KEY, name) == 0)
-                    && (-1 == frame->root->pid)) {
+    if (cluster_handle_marker_getxattr(frame, loc, name, priv->vol_uuid,
+                                       afr_getxattr_unwind,
+                                       afr_marker_populate_args) == 0)
+        return 0;
 
-                        local->marker.call_count = priv->child_count;
+    ret = afr_handle_heal_xattrs(frame, this, &local->loc, name);
+    if (ret == 0)
+        return 0;
 
-                        sub_volumes = alloca ( priv->child_count * sizeof (xlator_t *));
-                        for (i = 0, trav = this->children; trav ;
-                                                trav = trav->next, i++) {
+    /*
+     * Heal daemons don't have IO threads ... and as a result they
+     * send this getxattr down and eventually crash :(
+     */
+    op_errno = -1;
+    GF_CHECK_XATTR_KEY_AND_GOTO(name, IO_THREADS_QUEUE_SIZE_KEY, op_errno, out);
+
+    /*
+     * Special xattrs which need responses from all subvols
+     */
+    if (afr_is_special_xattr(name, &cbk, 0)) {
+        afr_getxattr_all_subvols(this, frame, name, loc, cbk);
+        return 0;
+    }
 
-                                *(sub_volumes + i)  = trav->xlator;
-                        }
+    if (XATTR_IS_NODE_UUID(name)) {
+        i = 0;
+        STACK_WIND_COOKIE(frame, afr_getxattr_node_uuid_cbk, (void *)(long)i,
+                          children[i], children[i]->fops->getxattr, loc, name,
+                          xdata);
+        return 0;
+    }
 
-                        if (cluster_getmarkerattr (frame, this, loc, name,
-                                                   local, afr_getxattr_unwind,
-                                                   sub_volumes,
-                                                   priv->child_count,
-                                                   MARKER_UUID_TYPE,
-                                                   priv->vol_uuid)) {
+no_name:
 
-                                op_errno = EINVAL;
-                                goto out;
-                       }
+    afr_read_txn(frame, this, local->loc.inode, afr_getxattr_wind,
+                 AFR_METADATA_TRANSACTION);
 
-                        return 0;
-                }
+    ret = 0;
+out:
+    if (ret < 0)
+        AFR_STACK_UNWIND(getxattr, frame, -1, op_errno, NULL, NULL);
+    return 0;
+}
 
-                if (*priv->vol_uuid) {
-                        if ((match_uuid_local (name, priv->vol_uuid) == 0)
-                            && (-1 == frame->root->pid)) {
+/* {{{ fgetxattr */
 
-                                local->marker.call_count = priv->child_count;
+int32_t
+afr_fgetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
 
-                                sub_volumes = alloca ( priv->child_count * sizeof (xlator_t *));
-                                for (i = 0, trav = this->children; trav ;
-                                                trav = trav->next, i++) {
+    local = frame->local;
 
-                                        *(sub_volumes + i)  = trav->xlator;
+    if (op_ret < 0) {
+        local->op_ret = -1;
+        local->op_errno = op_errno;
 
-                                }
+        afr_read_txn_continue(frame, this, (long)cookie);
+        return 0;
+    }
 
-                                if (cluster_getmarkerattr (frame, this, loc,
-                                                           name, local,
-                                                           afr_getxattr_unwind,
-                                                           sub_volumes,
-                                                           priv->child_count,
-                                                           MARKER_XTIME_TYPE,
-                                                           priv->vol_uuid)) {
-                                        op_errno = EINVAL;
-                                        goto out;
-                                }
+    if (dict)
+        afr_filter_xattrs(dict);
 
-                                return 0;
-                        }
-                }
+    AFR_STACK_UNWIND(fgetxattr, frame, op_ret, op_errno, dict, xdata);
 
-        }
+    return 0;
+}
 
-        read_child = afr_read_child (this, loc->inode);
+int
+afr_fgetxattr_wind(call_frame_t *frame, xlator_t *this, int subvol)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        if ((read_child >= 0) && (priv->child_up[read_child])) {
-                call_child = read_child;
+    local = frame->local;
+    priv = this->private;
 
-                local->cont.getxattr.last_tried = -1;
-        } else {
-                call_child = afr_first_up_child (priv);
+    if (subvol == -1) {
+        AFR_STACK_UNWIND(fgetxattr, frame, local->op_ret, local->op_errno, NULL,
+                         NULL);
+        return 0;
+    }
 
-                if (call_child == -1) {
-                        op_errno = ENOTCONN;
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "no child is up");
-                        goto out;
-                }
+    STACK_WIND_COOKIE(frame, afr_fgetxattr_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->fgetxattr, local->fd,
+                      local->cont.getxattr.name, local->xdata_req);
+    return 0;
+}
 
-                local->cont.getxattr.last_tried = call_child;
+static void
+afr_fgetxattr_all_subvols(xlator_t *this, call_frame_t *frame,
+                          fop_fgetxattr_cbk_t cbk)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int i = 0;
+    int call_count = 0;
+
+    priv = this->private;
+
+    local = frame->local;
+    // local->call_count set in afr_local_init
+    call_count = local->call_count;
+
+    // If up-children count is 0, afr_local_init would have failed already
+    // and the call would have unwound so not handling it here.
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->child_up[i]) {
+            STACK_WIND_COOKIE(frame, cbk, (void *)(long)i, priv->children[i],
+                              priv->children[i]->fops->fgetxattr, local->fd,
+                              local->cont.getxattr.name, NULL);
+            if (!--call_count)
+                break;
         }
+    }
 
+    return;
+}
 
-	STACK_WIND_COOKIE (frame, afr_getxattr_cbk,
-			   (void *) (long) call_child,
-			   children[call_child], children[call_child]->fops->getxattr,
-			   loc, name);
+int
+afr_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
+              dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    int32_t op_errno = 0;
+    fop_fgetxattr_cbk_t cbk = NULL;
+
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
+
+    local->op = GF_FOP_FGETXATTR;
+    local->fd = fd_ref(fd);
+    if (name) {
+        local->cont.getxattr.name = gf_strdup(name);
+        if (!local->cont.getxattr.name) {
+            op_errno = ENOMEM;
+            goto out;
+        }
+    }
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
+
+    /* pathinfo gets handled only in getxattr(), but we need to handle
+     * lockinfo.
+     * If we are doing fgetxattr with lockinfo as the key then we
+     * collect information from all children.
+     */
+    if (afr_is_special_xattr(name, &cbk, 1)) {
+        afr_fgetxattr_all_subvols(this, frame, cbk);
+        return 0;
+    }
+
+    afr_fix_open(fd, this);
+
+    afr_read_txn(frame, this, fd->inode, afr_fgetxattr_wind,
+                 AFR_METADATA_TRANSACTION);
 
-	op_ret = 0;
+    return 0;
 out:
-	if (op_ret == -1) {
-		AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, NULL);
-	}
-	return 0;
-}
+    AFR_STACK_UNWIND(fgetxattr, frame, -1, op_errno, NULL, NULL);
 
+    return 0;
+}
 
 /* }}} */
 
 /* {{{ readv */
 
-/**
- * read algorithm:
- *
- * if the user has specified a read subvolume, use it
- * otherwise -
- *   use the inode number to hash it to one of the subvolumes, and
- *   read from there (to balance read load)
- *
- * if any of the above read's fail, try the children in sequence
- * beginning at the beginning
- */
-
-int32_t
-afr_readv_cbk (call_frame_t *frame, void *cookie,
-	       xlator_t *this, int32_t op_ret, int32_t op_errno,
-	       struct iovec *vector, int32_t count, struct iatt *buf,
-               struct iobref *iobref)
+int
+afr_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct iovec *vector, int32_t count,
+              struct iatt *buf, struct iobref *iobref, dict_t *xdata)
 {
-	afr_private_t * priv     = NULL;
-	afr_local_t *   local    = NULL;
-	xlator_t **     children = NULL;
+    afr_local_t *local = NULL;
 
-	int unwind     = 1;
-	int last_tried = -1;
-	int this_try   = -1;
-        int read_child = -1;
+    local = frame->local;
 
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
+    if (op_ret < 0) {
+        local->op_ret = -1;
+        local->op_errno = op_errno;
 
-	priv     = this->private;
-	VALIDATE_OR_GOTO (priv->children, out);
+        afr_read_txn_continue(frame, this, (long)cookie);
+        return 0;
+    }
+
+    AFR_STACK_UNWIND(readv, frame, op_ret, op_errno, vector, count, buf, iobref,
+                     xdata);
+    return 0;
+}
 
-	children = priv->children;
+int
+afr_readv_wind(call_frame_t *frame, xlator_t *this, int subvol)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-	local = frame->local;
+    local = frame->local;
+    priv = this->private;
 
-        read_child = (long) cookie;
+    if (subvol == -1) {
+        AFR_STACK_UNWIND(readv, frame, local->op_ret, local->op_errno, 0, 0, 0,
+                         0, 0);
+        return 0;
+    }
 
-	if (op_ret == -1) {
-	retry:
-		last_tried = local->cont.readv.last_tried;
+    STACK_WIND_COOKIE(
+        frame, afr_readv_cbk, (void *)(long)subvol, priv->children[subvol],
+        priv->children[subvol]->fops->readv, local->fd, local->cont.readv.size,
+        local->cont.readv.offset, local->cont.readv.flags, local->xdata_req);
+    return 0;
+}
 
-		if (all_tried (last_tried, priv->child_count)) {
-			goto out;
-		}
-		this_try = ++local->cont.readv.last_tried;
+int
+afr_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+          off_t offset, uint32_t flags, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    int32_t op_errno = 0;
 
-		if (this_try == read_child) {
-			/*
-			   skip the read child since if we are here
-			   we must have already tried that child
-			*/
-			goto retry;
-		}
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-		unwind = 0;
+    local->op = GF_FOP_READ;
+    local->fd = fd_ref(fd);
+    local->cont.readv.size = size;
+    local->cont.readv.offset = offset;
+    local->cont.readv.flags = flags;
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
 
-		STACK_WIND_COOKIE (frame, afr_readv_cbk,
-				   (void *) (long) read_child,
-				   children[this_try],
-				   children[this_try]->fops->readv,
-				   local->fd, local->cont.readv.size,
-				   local->cont.readv.offset);
-	}
+    afr_fix_open(fd, this);
 
-out:
-	if (unwind) {
-                if (buf && local)
-                        buf->ia_ino = local->cont.readv.ino;
+    afr_read_txn(frame, this, fd->inode, afr_readv_wind, AFR_DATA_TRANSACTION);
 
-		AFR_STACK_UNWIND (readv, frame, op_ret, op_errno,
-                                  vector, count, buf, iobref);
-	}
+    return 0;
+out:
+    AFR_STACK_UNWIND(readv, frame, -1, op_errno, 0, 0, 0, 0, 0);
 
-	return 0;
+    return 0;
 }
 
+/* }}} */
 
-int32_t
-afr_readv (call_frame_t *frame, xlator_t *this,
-	   fd_t *fd, size_t size, off_t offset)
+/* {{{ seek */
+
+int
+afr_seek_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, off_t offset, dict_t *xdata)
 {
-	afr_private_t * priv       = NULL;
-	afr_local_t   * local      = NULL;
-	xlator_t **     children   = NULL;
+    afr_local_t *local = NULL;
 
-        int32_t         read_child = -1;
-	int             call_child = 0;
+    local = frame->local;
 
-	int32_t         op_ret     = -1;
-	int32_t         op_errno   = 0;
+    if (op_ret < 0) {
+        local->op_ret = -1;
+        local->op_errno = op_errno;
 
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
-	VALIDATE_OR_GOTO (fd, out);
+        afr_read_txn_continue(frame, this, (long)cookie);
+        return 0;
+    }
 
-	priv     = this->private;
-	children = priv->children;
+    AFR_STACK_UNWIND(seek, frame, op_ret, op_errno, offset, xdata);
+    return 0;
+}
 
-	ALLOC_OR_GOTO (local, afr_local_t, out);
+int
+afr_seek_wind(call_frame_t *frame, xlator_t *this, int subvol)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-	frame->local = local;
+    local = frame->local;
+    priv = this->private;
 
-        read_child = afr_read_child (this, fd->inode);
+    if (subvol == -1) {
+        AFR_STACK_UNWIND(seek, frame, local->op_ret, local->op_errno, 0, NULL);
+        return 0;
+    }
 
-        if ((read_child >= 0) && (priv->child_up[read_child])) {
-                call_child = read_child;
+    STACK_WIND_COOKIE(
+        frame, afr_seek_cbk, (void *)(long)subvol, priv->children[subvol],
+        priv->children[subvol]->fops->seek, local->fd, local->cont.seek.offset,
+        local->cont.seek.what, local->xdata_req);
+    return 0;
+}
 
-		/*
-		   if read fails from the read child, we try
-		   all children starting with the first one
-		*/
-                local->cont.readv.last_tried = -1;
+int
+afr_seek(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+         gf_seek_what_t what, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    int32_t op_errno = 0;
 
-        } else {
-		call_child = afr_first_up_child (priv);
-		if (call_child == -1) {
-			op_errno = ENOTCONN;
-			gf_log (this->name, GF_LOG_DEBUG,
-				"no child is up");
-			goto out;
-		}
-
-		local->cont.readv.last_tried = call_child;
-	}
-
-	local->fd                    = fd_ref (fd);
-
-        local->cont.readv.ino        = fd->inode->ino;
-	local->cont.readv.size       = size;
-	local->cont.readv.offset     = offset;
-
-	STACK_WIND_COOKIE (frame, afr_readv_cbk,
-			   (void *) (long) call_child,
-			   children[call_child],
-			   children[call_child]->fops->readv,
-			   fd, size, offset);
-
-	op_ret = 0;
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
+
+    local->op = GF_FOP_SEEK;
+    local->fd = fd_ref(fd);
+    local->cont.seek.offset = offset;
+    local->cont.seek.what = what;
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
+
+    afr_fix_open(fd, this);
+
+    afr_read_txn(frame, this, fd->inode, afr_seek_wind, AFR_DATA_TRANSACTION);
+
+    return 0;
 out:
-	if (op_ret == -1) {
-		AFR_STACK_UNWIND (readv, frame, op_ret, op_errno, NULL, 0, NULL,
-                                  NULL);
-	}
-	return 0;
-}
+    AFR_STACK_UNWIND(seek, frame, -1, op_errno, 0, NULL);
 
+    return 0;
+}
 /* }}} */
diff --git a/xlators/cluster/afr/src/afr-inode-read.h b/xlators/cluster/afr/src/afr-inode-read.h
index acc814fb7dc..8c982bc7e6f 100644
--- a/xlators/cluster/afr/src/afr-inode-read.h
+++ b/xlators/cluster/afr/src/afr-inode-read.h
@@ -1,47 +1,45 @@
 /*
-   Copyright (c) 2007-2010 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __INODE_READ_H__
 #define __INODE_READ_H__
 
 int32_t
-afr_access (call_frame_t *frame, xlator_t *this,
-	    loc_t *loc, int32_t mask);
+afr_access(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+           dict_t *xdata);
 
 int32_t
-afr_stat (call_frame_t *frame, xlator_t *this,
-	  loc_t *loc);
+afr_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata);
 
 int32_t
-afr_fstat (call_frame_t *frame, xlator_t *this,
-	   fd_t *fd);
+afr_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata);
 
 int32_t
-afr_readlink (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, size_t size);
+afr_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+             dict_t *xdata);
 
 int32_t
-afr_readv (call_frame_t *frame, xlator_t *this,
-	   fd_t *fd, size_t size, off_t offset);
+afr_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+          off_t offset, uint32_t flags, dict_t *xdata);
 
 int32_t
-afr_getxattr (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, const char *name);
+afr_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, const char *name,
+             dict_t *xdata);
 
+int32_t
+afr_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
+              dict_t *xdata);
+
+int
+afr_seek(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+         gf_seek_what_t what, dict_t *xdata);
+int
+afr_handle_quota_size(call_frame_t *frame, xlator_t *this);
 #endif /* __INODE_READ_H__ */
diff --git a/xlators/cluster/afr/src/afr-inode-write.c b/xlators/cluster/afr/src/afr-inode-write.c
index 050a4f0e9ef..1d6e4f3570a 100644
--- a/xlators/cluster/afr/src/afr-inode-write.c
+++ b/xlators/cluster/afr/src/afr-inode-write.c
@@ -1,1594 +1,2565 @@
 /*
-   Copyright (c) 2007-2010 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
 
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
 
-#include <libgen.h>
 #include <unistd.h>
-#include <fnmatch.h>
 #include <sys/time.h>
 #include <stdlib.h>
 #include <signal.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "afr.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "stack.h"
-#include "list.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include "compat-errno.h"
-#include "compat.h"
-
+#include <glusterfs/glusterfs.h>
 #include "afr.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/compat.h>
+#include "protocol-common.h"
+#include <glusterfs/byte-order.h>
 #include "afr-transaction.h"
+#include "afr-self-heal.h"
+#include "afr-messages.h"
 
-/* {{{ writev */
+static void
+__afr_inode_write_finalize(call_frame_t *frame, xlator_t *this)
+{
+    int i = 0;
+    int ret = 0;
+    int read_subvol = 0;
+    struct iatt *stbuf = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    afr_read_subvol_args_t args = {
+        0,
+    };
+
+    local = frame->local;
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, local->inode, out);
+
+    /*This code needs to stay till DHT sends fops on linked
+     * inodes*/
+    if (!inode_is_linked(local->inode)) {
+        for (i = 0; i < priv->child_count; i++) {
+            if (!local->replies[i].valid)
+                continue;
+            if (local->replies[i].op_ret == -1)
+                continue;
+            if (!gf_uuid_is_null(local->replies[i].poststat.ia_gfid)) {
+                gf_uuid_copy(args.gfid, local->replies[i].poststat.ia_gfid);
+                args.ia_type = local->replies[i].poststat.ia_type;
+                break;
+            } else {
+                ret = dict_get_bin(local->replies[i].xdata,
+                                   DHT_IATT_IN_XDATA_KEY, (void **)&stbuf);
+                if (ret)
+                    continue;
+                gf_uuid_copy(args.gfid, stbuf->ia_gfid);
+                args.ia_type = stbuf->ia_type;
+                break;
+            }
+        }
+    }
+
+    if (local->transaction.type == AFR_METADATA_TRANSACTION) {
+        read_subvol = afr_metadata_subvol_get(local->inode, this, NULL,
+                                              local->readable, NULL, &args);
+    } else {
+        read_subvol = afr_data_subvol_get(local->inode, this, NULL,
+                                          local->readable, NULL, &args);
+    }
+
+    local->op_ret = -1;
+    local->op_errno = afr_final_errno(local, priv);
+    afr_pick_error_xdata(local, priv, local->inode, local->readable, NULL,
+                         NULL);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid)
+            continue;
+        if (local->replies[i].op_ret < 0)
+            continue;
+
+        /* Order of checks in the compound conditional
+           below is important.
+
+           - Highest precedence: largest op_ret
+           - Next precedence: if all op_rets are equal, read subvol
+           - Least precedence: any succeeded subvol
+        */
+        if ((local->op_ret < local->replies[i].op_ret) ||
+            ((local->op_ret == local->replies[i].op_ret) &&
+             (i == read_subvol))) {
+            local->op_ret = local->replies[i].op_ret;
+            local->op_errno = local->replies[i].op_errno;
+
+            local->cont.inode_wfop.prebuf = local->replies[i].prestat;
+            local->cont.inode_wfop.postbuf = local->replies[i].poststat;
+
+            if (local->replies[i].xdata) {
+                if (local->xdata_rsp)
+                    dict_unref(local->xdata_rsp);
+                local->xdata_rsp = dict_ref(local->replies[i].xdata);
+            }
+            if (local->replies[i].xattr) {
+                if (local->xattr_rsp)
+                    dict_unref(local->xattr_rsp);
+                local->xattr_rsp = dict_ref(local->replies[i].xattr);
+            }
+        }
+    }
+
+    afr_set_in_flight_sb_status(this, frame, local->inode);
+out:
+    return;
+}
 
-int
-afr_writev_unwind (call_frame_t *frame, xlator_t *this)
+static void
+__afr_inode_write_fill(call_frame_t *frame, xlator_t *this, int child_index,
+                       int op_ret, int op_errno, struct iatt *prebuf,
+                       struct iatt *postbuf, dict_t *xattr, dict_t *xdata)
 {
-        afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    local->replies[child_index].valid = 1;
+
+    if (AFR_IS_ARBITER_BRICK(priv, child_index) && op_ret == 1)
+        op_ret = iov_length(local->cont.writev.vector,
+                            local->cont.writev.count);
+
+    local->replies[child_index].op_ret = op_ret;
+    local->replies[child_index].op_errno = op_errno;
+    if (xdata)
+        local->replies[child_index].xdata = dict_ref(xdata);
+
+    if (op_ret >= 0) {
+        if (prebuf)
+            local->replies[child_index].prestat = *prebuf;
+        if (postbuf)
+            local->replies[child_index].poststat = *postbuf;
+        if (xattr)
+            local->replies[child_index].xattr = dict_ref(xattr);
+    } else {
+        afr_transaction_fop_failed(frame, this, child_index);
+    }
+
+    return;
+}
 
-        local = frame->local;
+static int
+__afr_inode_write_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xattr, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    int child_index = (long)cookie;
+    int call_count = -1;
+    afr_private_t *priv = NULL;
 
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame)
-                        main_frame = local->transaction.main_frame;
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
+    priv = this->private;
+    local = frame->local;
 
-        if (main_frame) {
-                local->cont.writev.prebuf.ia_ino  = local->cont.writev.ino;
-                local->cont.writev.postbuf.ia_ino = local->cont.writev.ino;
+    LOCK(&frame->lock);
+    {
+        __afr_inode_write_fill(frame, this, child_index, op_ret, op_errno,
+                               prebuf, postbuf, xattr, xdata);
+        call_count = --local->call_count;
+    }
+    UNLOCK(&frame->lock);
 
-                AFR_STACK_UNWIND (writev, main_frame,
-                                  local->op_ret, local->op_errno,
-                                  &local->cont.writev.prebuf,
-                                  &local->cont.writev.postbuf);
+    if (call_count == 0) {
+        __afr_inode_write_finalize(frame, this);
+
+        if (afr_txn_nothing_failed(frame, this)) {
+            /*if it did pre-op, it will do post-op changing ctime*/
+            if (priv->consistent_metadata && afr_needs_changelog_update(local))
+                afr_zero_fill_stat(local);
+            local->transaction.unwind(frame, this);
         }
-        return 0;
+
+        afr_transaction_resume(frame, this);
+    }
+
+    return 0;
 }
 
+/* {{{ writev */
 
-int
-afr_writev_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                     struct iatt *postbuf)
+void
+afr_writev_copy_outvars(call_frame_t *src_frame, call_frame_t *dst_frame)
 {
-        afr_local_t *   local = NULL;
-
-        int child_index = (long) cookie;
-        int call_count  = -1;
-        int read_child  = 0;
+    afr_local_t *src_local = NULL;
+    afr_local_t *dst_local = NULL;
+
+    src_local = src_frame->local;
+    dst_local = dst_frame->local;
+
+    dst_local->op_ret = src_local->op_ret;
+    dst_local->op_errno = src_local->op_errno;
+    dst_local->cont.inode_wfop.prebuf = src_local->cont.inode_wfop.prebuf;
+    dst_local->cont.inode_wfop.postbuf = src_local->cont.inode_wfop.postbuf;
+    if (src_local->xdata_rsp)
+        dst_local->xdata_rsp = dict_ref(src_local->xdata_rsp);
+}
 
-        local = frame->local;
+void
+afr_writev_unwind(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = this->private;
 
-        read_child = afr_read_child (this, local->fd->inode);
+    local = frame->local;
 
-        LOCK (&frame->lock);
-        {
-                if (child_index == read_child) {
-                        local->read_child_returned = _gf_true;
-                }
+    if (priv->consistent_metadata)
+        afr_zero_fill_stat(local);
 
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
+    AFR_STACK_UNWIND(writev, frame, local->op_ret, local->op_errno,
+                     &local->cont.inode_wfop.prebuf,
+                     &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+}
 
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret              = op_ret;
-                                local->cont.writev.prebuf  = *prebuf;
-                                local->cont.writev.postbuf = *postbuf;
-                        }
+int
+afr_transaction_writev_unwind(call_frame_t *frame, xlator_t *this)
+{
+    call_frame_t *fop_frame = NULL;
 
-                        if (child_index == read_child) {
-                                local->cont.writev.prebuf  = *prebuf;
-                                local->cont.writev.postbuf = *postbuf;
-                        }
-                }
+    fop_frame = afr_transaction_detach_fop_frame(frame);
 
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
+    if (fop_frame) {
+        afr_writev_copy_outvars(frame, fop_frame);
+        afr_writev_unwind(fop_frame, this);
+    }
+    return 0;
+}
 
-        if (call_count == 0) {
-                local->transaction.unwind (frame, this);
+static void
+afr_writev_handle_short_writes(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+
+    local = frame->local;
+    priv = this->private;
+    /*
+     * We already have the best case result of the writev calls staged
+     * as the return value. Any writev that returns some value less
+     * than the best case is now out of sync, so mark the fop as
+     * failed. Note that fops that have returned with errors have
+     * already been marked as failed.
+     */
+    for (i = 0; i < priv->child_count; i++) {
+        if ((!local->replies[i].valid) || (local->replies[i].op_ret == -1))
+            continue;
+
+        if (local->replies[i].op_ret < local->op_ret)
+            afr_transaction_fop_failed(frame, this, i);
+    }
+}
 
-                local->transaction.resume (frame, this);
-        }
-        return 0;
+void
+afr_inode_write_fill(call_frame_t *frame, xlator_t *this, int child_index,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
+{
+    int ret = 0;
+    afr_local_t *local = frame->local;
+    uint32_t open_fd_count = 0;
+    uint32_t write_is_append = 0;
+    int32_t num_inodelks = 0;
+
+    LOCK(&frame->lock);
+    {
+        __afr_inode_write_fill(frame, this, child_index, op_ret, op_errno,
+                               prebuf, postbuf, NULL, xdata);
+        if (op_ret == -1 || !xdata)
+            goto unlock;
+
+        write_is_append = 0;
+        ret = dict_get_uint32(xdata, GLUSTERFS_WRITE_IS_APPEND,
+                              &write_is_append);
+        if (ret || !write_is_append)
+            local->append_write = _gf_false;
+
+        ret = dict_get_uint32(xdata, GLUSTERFS_ACTIVE_FD_COUNT, &open_fd_count);
+        if (ret < 0)
+            goto unlock;
+        if (open_fd_count > local->open_fd_count) {
+            local->open_fd_count = open_fd_count;
+            local->update_open_fd_count = _gf_true;
+        }
+
+        ret = dict_get_int32_sizen(xdata, GLUSTERFS_INODELK_COUNT,
+                                   &num_inodelks);
+        if (ret < 0)
+            goto unlock;
+        if (num_inodelks > local->num_inodelks) {
+            local->num_inodelks = num_inodelks;
+            local->update_num_inodelks = _gf_true;
+        }
+    }
+unlock:
+    UNLOCK(&frame->lock);
 }
 
+void
+afr_process_post_writev(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_lock_t *lock = NULL;
+
+    local = frame->local;
+
+    if (!local->stable_write && !local->append_write)
+        /* An appended write removes the necessity to
+           fsync() the file. This is because self-heal
+           has the logic to check for larger file when
+           the xattrs are not reliably pointing at
+           a stale file.
+        */
+        afr_fd_report_unstable_write(this, local);
+
+    __afr_inode_write_finalize(frame, this);
+
+    afr_writev_handle_short_writes(frame, this);
+
+    if (local->update_open_fd_count)
+        local->inode_ctx->open_fd_count = local->open_fd_count;
+    if (local->update_num_inodelks &&
+        local->transaction.type == AFR_DATA_TRANSACTION) {
+        lock = &local->inode_ctx->lock[local->transaction.type];
+        lock->num_inodelks = local->num_inodelks;
+    }
+}
 
 int
-afr_writev_wind (call_frame_t *frame, xlator_t *this)
+afr_writev_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                    struct iatt *postbuf, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    call_frame_t *fop_frame = NULL;
+    int child_index = (long)cookie;
+    int call_count = -1;
 
-        int i = 0;
-        int call_count = -1;
+    afr_inode_write_fill(frame, this, child_index, op_ret, op_errno, prebuf,
+                         postbuf, xdata);
 
-        local = frame->local;
-        priv = this->private;
+    call_count = afr_frame_return(frame);
 
-        call_count = afr_up_children_count (priv->child_count, local->child_up);
+    if (call_count == 0) {
+        afr_process_post_writev(frame, this);
 
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
-
-        local->call_count = call_count;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, afr_writev_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->writev,
-                                           local->fd,
-                                           local->cont.writev.vector,
-                                           local->cont.writev.count,
-                                           local->cont.writev.offset,
-                                           local->cont.writev.iobref);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
-
-        return 0;
+        if (!afr_txn_nothing_failed(frame, this)) {
+            // Don't unwind until post-op is complete
+            afr_transaction_resume(frame, this);
+        } else {
+            /*
+             * Generally inode-write fops do transaction.unwind then
+             * transaction.resume, but writev needs to make sure that
+             * delayed post-op frame is placed in fdctx before unwind
+             * happens. This prevents the race of flush doing the
+             * changelog wakeup first in fuse thread and then this
+             * writev placing its delayed post-op frame in fdctx.
+             * This helps flush make sure all the delayed post-ops are
+             * completed.
+             */
+
+            fop_frame = afr_transaction_detach_fop_frame(frame);
+            afr_writev_copy_outvars(frame, fop_frame);
+            afr_transaction_resume(frame, this);
+            afr_writev_unwind(fop_frame, this);
+        }
+    }
+    return 0;
 }
 
+static int
+afr_arbiter_writev_wind(call_frame_t *frame, xlator_t *this, int subvol)
+{
+    afr_local_t *local = frame->local;
+    afr_private_t *priv = this->private;
+    static char byte = 0xFF;
+    static struct iovec vector = {&byte, 1};
+    int32_t count = 1;
+
+    STACK_WIND_COOKIE(
+        frame, afr_writev_wind_cbk, (void *)(long)subvol,
+        priv->children[subvol], priv->children[subvol]->fops->writev, local->fd,
+        &vector, count, local->cont.writev.offset, local->cont.writev.flags,
+        local->cont.writev.iobref, local->xdata_req);
+
+    return 0;
+}
 
 int
-afr_writev_done (call_frame_t *frame, xlator_t *this)
+afr_writev_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-
-        local = frame->local;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        iobref_unref (local->cont.writev.iobref);
-        local->cont.writev.iobref = NULL;
-
-        local->transaction.unwind (frame, this);
-
-        AFR_STACK_DESTROY (frame);
+    local = frame->local;
+    priv = this->private;
 
+    if (AFR_IS_ARBITER_BRICK(priv, subvol)) {
+        afr_arbiter_writev_wind(frame, this, subvol);
         return 0;
+    }
+
+    STACK_WIND_COOKIE(frame, afr_writev_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->writev, local->fd,
+                      local->cont.writev.vector, local->cont.writev.count,
+                      local->cont.writev.offset, local->cont.writev.flags,
+                      local->cont.writev.iobref, local->xdata_req);
+    return 0;
 }
 
+int
+afr_do_writev(call_frame_t *frame, xlator_t *this)
+{
+    call_frame_t *transaction_frame = NULL;
+    afr_local_t *local = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
+
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
+
+    local = frame->local;
+    transaction_frame->local = local;
+    frame->local = NULL;
+
+    if (!AFR_FRAME_INIT(frame, op_errno))
+        goto out;
+
+    local->op = GF_FOP_WRITE;
+
+    local->transaction.wind = afr_writev_wind;
+    local->transaction.unwind = afr_transaction_writev_unwind;
+
+    local->transaction.main_frame = frame;
+
+    if (local->fd->flags & O_APPEND) {
+        /*
+         * Backend vfs ignores the 'offset' for append mode fd so
+         * locking just the region provided for the writev does not
+         * give consistency guarantee. The actual write may happen at a
+         * completely different range than the one provided by the
+         * offset, len in the fop. So lock the entire file.
+         */
+        local->transaction.start = 0;
+        local->transaction.len = 0;
+    } else {
+        local->transaction.start = local->cont.writev.offset;
+        local->transaction.len = iov_length(local->cont.writev.vector,
+                                            local->cont.writev.count);
+    }
+
+    ret = afr_transaction(transaction_frame, this, AFR_DATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
+out:
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
+
+    AFR_STACK_UNWIND(writev, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
+}
 
 int
-afr_do_writev (call_frame_t *frame, xlator_t *this)
+afr_writev(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
+           int32_t count, off_t offset, uint32_t flags, struct iobref *iobref,
+           dict_t *xdata)
 {
-        call_frame_t * transaction_frame = NULL;
-        afr_local_t *  local             = NULL;
+    afr_local_t *local = NULL;
+    int op_errno = ENOMEM;
+    int ret = -1;
+
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
+
+    local->cont.writev.vector = iov_dup(vector, count);
+    if (!local->cont.writev.vector)
+        goto out;
+    local->cont.writev.count = count;
+    local->cont.writev.offset = offset;
+    local->cont.writev.flags = flags;
+    local->cont.writev.iobref = iobref_ref(iobref);
+
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
+
+    if (!local->xdata_req)
+        goto out;
+
+    local->fd = fd_ref(fd);
+    ret = afr_set_inode_local(this, local, fd->inode);
+    if (ret)
+        goto out;
+
+    if (dict_set_uint32(local->xdata_req, GLUSTERFS_ACTIVE_FD_COUNT, 4)) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    if (dict_set_str_sizen(local->xdata_req, GLUSTERFS_INODELK_DOM_COUNT,
+                           this->name)) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    if (dict_set_uint32(local->xdata_req, GLUSTERFS_WRITE_IS_APPEND, 4)) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    /* Set append_write to be true speculatively. If on any
+       server it turns not be true, we unset it in the
+       callback.
+    */
+    local->append_write = _gf_true;
+
+    /* detect here, but set it in writev_wind_cbk *after* the unstable
+       write is performed
+    */
+    local->stable_write = !!((fd->flags | flags) & (O_SYNC | O_DSYNC));
+
+    afr_fix_open(fd, this);
+
+    afr_do_writev(frame, this);
+
+    return 0;
+out:
+    AFR_STACK_UNWIND(writev, frame, -1, op_errno, NULL, NULL, NULL);
 
-        int op_ret   = -1;
-        int op_errno = 0;
+    return 0;
+}
 
-        local = frame->local;
+/* }}} */
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory.");
-                op_errno = ENOMEM;
-                goto out;
-        }
+/* {{{ truncate */
 
-        transaction_frame->local = local;
-        frame->local = NULL;
+int
+afr_truncate_unwind(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-        local->op = GF_FOP_WRITE;
+    local = frame->local;
 
-        local->success_count      = 0;
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
+        return 0;
 
-        local->transaction.fop    = afr_writev_wind;
-        local->transaction.done   = afr_writev_done;
-        local->transaction.unwind = afr_writev_unwind;
+    AFR_STACK_UNWIND(truncate, main_frame, local->op_ret, local->op_errno,
+                     &local->cont.inode_wfop.prebuf,
+                     &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+    return 0;
+}
 
-        local->transaction.main_frame = frame;
-        if (local->fd->flags & O_APPEND) {
-                local->transaction.start   = 0;
-                local->transaction.len     = 0;
-        } else {
-                local->transaction.start   = local->cont.writev.offset;
-                local->transaction.len     = iov_length (local->cont.writev.vector,
-                                                         local->cont.writev.count);
-        }
+int
+afr_truncate_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
 
-        afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
+    local = frame->local;
 
-        op_ret = 0;
-out:
-        if (op_ret == -1) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (writev, frame, op_ret, op_errno, NULL, NULL);
-        }
+    if (op_ret == 0 && prebuf->ia_size != postbuf->ia_size)
+        local->stable_write = _gf_false;
 
-        return 0;
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, prebuf,
+                                 postbuf, NULL, xdata);
 }
 
-
 int
-afr_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
-            struct iovec *vector, int32_t count, off_t offset,
-            struct iobref *iobref)
+afr_truncate_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        int ret = -1;
+    local = frame->local;
+    priv = this->private;
 
-        int op_ret   = -1;
-        int op_errno = 0;
+    STACK_WIND_COOKIE(frame, afr_truncate_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->truncate, &local->loc,
+                      local->cont.truncate.offset, local->xdata_req);
+    return 0;
+}
 
-        uint64_t ctx;
-        afr_fd_ctx_t *fd_ctx = NULL;
+int
+afr_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+             dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-        priv = this->private;
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        ALLOC_OR_GOTO (local, afr_local_t, out);
+    local->cont.truncate.offset = offset;
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
 
-        ret = AFR_LOCAL_INIT (local, priv);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
-        }
+    if (!local->xdata_req)
+        goto out;
 
-        frame->local = local;
+    local->transaction.wind = afr_truncate_wind;
+    local->transaction.unwind = afr_truncate_unwind;
 
-        local->cont.writev.vector     = iov_dup (vector, count);
-        local->cont.writev.count      = count;
-        local->cont.writev.offset     = offset;
-        local->cont.writev.ino        = fd->inode->ino;
-        local->cont.writev.iobref     = iobref_ref (iobref);
+    loc_copy(&local->loc, loc);
+    ret = afr_set_inode_local(this, local, loc->inode);
+    if (ret)
+        goto out;
 
-        local->fd                = fd_ref (fd);
+    local->op = GF_FOP_TRUNCATE;
 
-        ret = fd_ctx_get (fd, this, &ctx);
-        if (ret < 0) {
-                goto out;
-        }
+    local->transaction.main_frame = frame;
+    local->transaction.start = offset;
+    local->transaction.len = 0;
 
-        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
+    /* Set it true speculatively, will get reset in afr_truncate_wind_cbk
+       if truncate was not a NOP */
+    local->stable_write = _gf_true;
 
-        if (fd_ctx->up_count < priv->up_count) {
-                local->openfd_flush_cbk = afr_do_writev;
-                afr_openfd_flush (frame, this, fd);
-        } else {
-                afr_do_writev (frame, this);
-        }
+    ret = afr_transaction(transaction_frame, this, AFR_DATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
 
-        op_ret = 0;
+    return 0;
 out:
-        if (op_ret == -1) {
-                AFR_STACK_UNWIND (writev, frame, op_ret, op_errno, NULL, NULL);
-        }
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-        return 0;
+    AFR_STACK_UNWIND(truncate, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
-
 /* }}} */
 
-/* {{{ truncate */
+/* {{{ ftruncate */
 
 int
-afr_truncate_unwind (call_frame_t *frame, xlator_t *this)
+afr_ftruncate_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
-
-        local = frame->local;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame)
-                        main_frame = local->transaction.main_frame;
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
-
-        if (main_frame) {
-                local->cont.truncate.prebuf.ia_ino  = local->cont.truncate.ino;
-                local->cont.truncate.postbuf.ia_ino = local->cont.truncate.ino;
-
-                AFR_STACK_UNWIND (truncate, main_frame, local->op_ret,
-                                  local->op_errno,
-                                  &local->cont.truncate.prebuf,
-                                  &local->cont.truncate.postbuf);
-        }
+    local = frame->local;
 
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(ftruncate, main_frame, local->op_ret, local->op_errno,
+                     &local->cont.inode_wfop.prebuf,
+                     &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_truncate_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+afr_ftruncate_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                        int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                       struct iatt *postbuf)
-{
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
-
-        int child_index = (long) cookie;
-        int read_child  = 0;
-        int call_count  = -1;
-        int need_unwind = 0;
-
-        local = frame->local;
-        priv  = this->private;
-
-        read_child = afr_read_child (this, local->loc.inode);
-
-        LOCK (&frame->lock);
-        {
-                if (child_index == read_child) {
-                        local->read_child_returned = _gf_true;
-                }
-
-                if (afr_fop_failed (op_ret, op_errno) && op_errno != EFBIG)
-                        afr_transaction_fop_failed (frame, this, child_index);
-
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                                local->cont.truncate.prebuf  = *prebuf;
-                                local->cont.truncate.postbuf = *postbuf;
-                        }
-
-                        if (child_index == read_child) {
-                                local->cont.truncate.prebuf  = *prebuf;
-                                local->cont.truncate.postbuf = *postbuf;
-                        }
-
-                        local->success_count++;
-
-                        if ((local->success_count >= priv->wait_count)
-                            && local->read_child_returned) {
-                                need_unwind = 1;
-                        }
-                }
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
-
-        if (need_unwind)
-                local->transaction.unwind (frame, this);
+                       struct iatt *postbuf, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
 
-        call_count = afr_frame_return (frame);
+    local = frame->local;
 
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-        }
+    if (op_ret == 0 && prebuf->ia_size != postbuf->ia_size)
+        local->stable_write = _gf_false;
 
-        return 0;
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, prebuf,
+                                 postbuf, NULL, xdata);
 }
 
-
-int32_t
-afr_truncate_wind (call_frame_t *frame, xlator_t *this)
+int
+afr_ftruncate_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-
-        int call_count = -1;
-        int i = 0;
-
-        local = frame->local;
-        priv = this->private;
-
-        call_count = afr_up_children_count (priv->child_count, local->child_up);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
-
-        local->call_count = call_count;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, afr_truncate_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->truncate,
-                                           &local->loc,
-                                           local->cont.truncate.offset);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
+    local = frame->local;
+    priv = this->private;
 
-        return 0;
+    STACK_WIND_COOKIE(frame, afr_ftruncate_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->ftruncate, local->fd,
+                      local->cont.ftruncate.offset, local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_truncate_done (call_frame_t *frame, xlator_t *this)
+afr_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+              dict_t *xdata)
 {
-        afr_local_t *local = NULL;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
 
-        local = frame->local;
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-        local->transaction.unwind (frame, this);
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        AFR_STACK_DESTROY (frame);
-
-        return 0;
-}
+    local->cont.ftruncate.offset = offset;
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
 
+    if (!local->xdata_req)
+        goto out;
 
-int
-afr_truncate (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, off_t offset)
-{
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t   *transaction_frame = NULL;
+    local->fd = fd_ref(fd);
+    ret = afr_set_inode_local(this, local, fd->inode);
+    if (ret)
+        goto out;
 
-        int ret = -1;
+    local->op = GF_FOP_FTRUNCATE;
 
-        int op_ret   = -1;
-        int op_errno = 0;
+    local->transaction.wind = afr_ftruncate_wind;
+    local->transaction.unwind = afr_ftruncate_unwind;
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+    local->transaction.main_frame = frame;
 
-        priv = this->private;
+    local->transaction.start = local->cont.ftruncate.offset;
+    local->transaction.len = 0;
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory.");
-                goto out;
-        }
+    afr_fix_open(fd, this);
 
-        ALLOC_OR_GOTO (local, afr_local_t, out);
+    /* Set it true speculatively, will get reset in afr_ftruncate_wind_cbk
+       if truncate was not a NOP */
+    local->stable_write = _gf_true;
 
-        ret = AFR_LOCAL_INIT (local, priv);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
-        }
+    ret = afr_transaction(transaction_frame, this, AFR_DATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
 
-        transaction_frame->local = local;
+    return 0;
+out:
+    AFR_STACK_UNWIND(ftruncate, frame, -1, op_errno, NULL, NULL, NULL);
 
-        local->op_ret = -1;
+    return 0;
+}
 
-        local->cont.truncate.offset  = offset;
-        local->cont.truncate.ino     = loc->inode->ino;
+/* }}} */
 
-        local->transaction.fop    = afr_truncate_wind;
-        local->transaction.done   = afr_truncate_done;
-        local->transaction.unwind = afr_truncate_unwind;
+/* {{{ setattr */
 
-        loc_copy (&local->loc, loc);
+int
+afr_setattr_unwind(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-        local->transaction.main_frame = frame;
-        local->transaction.start   = 0;
-        local->transaction.len     = offset;
+    local = frame->local;
 
-        afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
+        return 0;
 
-        op_ret = 0;
-out:
-        if (op_ret == -1) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (truncate, frame, op_ret, op_errno, NULL, NULL);
-        }
+    AFR_STACK_UNWIND(setattr, main_frame, local->op_ret, local->op_errno,
+                     &local->cont.inode_wfop.prebuf,
+                     &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+    return 0;
+}
 
-        return 0;
+int
+afr_setattr_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, struct iatt *preop,
+                     struct iatt *postop, dict_t *xdata)
+{
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, preop,
+                                 postop, NULL, xdata);
 }
 
+int
+afr_setattr_wind(call_frame_t *frame, xlator_t *this, int subvol)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    STACK_WIND_COOKIE(frame, afr_setattr_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->setattr, &local->loc,
+                      &local->cont.setattr.in_buf, local->cont.setattr.valid,
+                      local->xdata_req);
+    return 0;
+}
 
-/* }}} */
+int
+afr_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *buf,
+            int32_t valid, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
+
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
+
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
+
+    local->cont.setattr.in_buf = *buf;
+    local->cont.setattr.valid = valid;
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
+
+    if (!local->xdata_req)
+        goto out;
+
+    local->transaction.wind = afr_setattr_wind;
+    local->transaction.unwind = afr_setattr_unwind;
+
+    loc_copy(&local->loc, loc);
+    ret = afr_set_inode_local(this, local, loc->inode);
+    if (ret)
+        goto out;
+
+    local->op = GF_FOP_SETATTR;
+
+    local->transaction.main_frame = frame;
+    local->transaction.start = LLONG_MAX - 1;
+    local->transaction.len = 0;
+
+    ret = afr_transaction(transaction_frame, this, AFR_METADATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
+out:
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-/* {{{ ftruncate */
+    AFR_STACK_UNWIND(setattr, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
+}
 
+/* {{{ fsetattr */
 
 int
-afr_ftruncate_unwind (call_frame_t *frame, xlator_t *this)
+afr_fsetattr_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame)
-                        main_frame = local->transaction.main_frame;
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
+        return 0;
 
-        if (main_frame) {
-                local->cont.ftruncate.prebuf.ia_ino  = local->cont.ftruncate.ino;
-                local->cont.ftruncate.postbuf.ia_ino = local->cont.ftruncate.ino;
+    AFR_STACK_UNWIND(fsetattr, main_frame, local->op_ret, local->op_errno,
+                     &local->cont.inode_wfop.prebuf,
+                     &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+    return 0;
+}
 
-                AFR_STACK_UNWIND (ftruncate, main_frame, local->op_ret,
-                                  local->op_errno,
-                                  &local->cont.ftruncate.prebuf,
-                                  &local->cont.ftruncate.postbuf);
-        }
-        return 0;
+int
+afr_fsetattr_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *preop,
+                      struct iatt *postop, dict_t *xdata)
+{
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, preop,
+                                 postop, NULL, xdata);
 }
 
+int
+afr_fsetattr_wind(call_frame_t *frame, xlator_t *this, int subvol)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    STACK_WIND_COOKIE(frame, afr_fsetattr_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->fsetattr, local->fd,
+                      &local->cont.fsetattr.in_buf, local->cont.fsetattr.valid,
+                      local->xdata_req);
+    return 0;
+}
 
 int
-afr_ftruncate_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                        struct iatt *postbuf)
+afr_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *buf,
+             int32_t valid, dict_t *xdata)
 {
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
 
-        int child_index = (long) cookie;
-        int call_count  = -1;
-        int need_unwind = 0;
-        int read_child  = 0;
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-        local = frame->local;
-        priv  = this->private;
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        read_child = afr_read_child (this, local->fd->inode);
+    local->cont.fsetattr.in_buf = *buf;
+    local->cont.fsetattr.valid = valid;
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
 
-        LOCK (&frame->lock);
-        {
-                if (child_index == read_child) {
-                        local->read_child_returned = _gf_true;
-                }
+    if (!local->xdata_req)
+        goto out;
 
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
+    local->transaction.wind = afr_fsetattr_wind;
+    local->transaction.unwind = afr_fsetattr_unwind;
 
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                                local->cont.ftruncate.prebuf  = *prebuf;
-                                local->cont.ftruncate.postbuf = *postbuf;
-                        }
+    local->fd = fd_ref(fd);
+    ret = afr_set_inode_local(this, local, fd->inode);
+    if (ret)
+        goto out;
 
-                        if (child_index == read_child) {
-                                local->cont.ftruncate.prebuf  = *prebuf;
-                                local->cont.ftruncate.postbuf = *postbuf;
-                        }
+    local->op = GF_FOP_FSETATTR;
 
-                        local->success_count++;
+    afr_fix_open(fd, this);
 
-                        if ((local->success_count >= priv->wait_count)
-                            && local->read_child_returned) {
-                                need_unwind = 1;
-                        }
-                }
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
+    local->transaction.main_frame = frame;
+    local->transaction.start = LLONG_MAX - 1;
+    local->transaction.len = 0;
 
-        if (need_unwind)
-                local->transaction.unwind (frame, this);
+    ret = afr_transaction(transaction_frame, this, AFR_METADATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
 
-        call_count = afr_frame_return (frame);
+    return 0;
+out:
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-        }
+    AFR_STACK_UNWIND(fsetattr, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
+}
+
+/* {{{ setxattr */
+
+int
+afr_setxattr_unwind(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
+    local = frame->local;
+
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
+
+    AFR_STACK_UNWIND(setxattr, main_frame, local->op_ret, local->op_errno,
+                     local->xdata_rsp);
+    return 0;
 }
 
+int
+afr_setxattr_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, NULL,
+                                 NULL, NULL, xdata);
+}
 
 int
-afr_ftruncate_wind (call_frame_t *frame, xlator_t *this)
+afr_setxattr_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    STACK_WIND_COOKIE(frame, afr_setxattr_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->setxattr, &local->loc,
+                      local->cont.setxattr.dict, local->cont.setxattr.flags,
+                      local->xdata_req);
+    return 0;
+}
 
-        int call_count = -1;
-        int i = 0;
+int
+afr_emptyb_set_pending_changelog_cbk(call_frame_t *frame, void *cookie,
+                                     xlator_t *this, int op_ret, int op_errno,
+                                     dict_t *xattr, dict_t *xdata)
 
-        local = frame->local;
-        priv = this->private;
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i, ret = 0;
+    char *op_type = NULL;
 
-        call_count = afr_up_children_count (priv->child_count, local->child_up);
+    local = frame->local;
+    priv = this->private;
+    i = (long)cookie;
 
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
+    local->replies[i].valid = 1;
+    local->replies[i].op_ret = op_ret;
+    local->replies[i].op_errno = op_errno;
 
-        local->call_count = call_count;
+    ret = dict_get_str_sizen(local->xdata_req, "replicate-brick-op", &op_type);
+    if (ret)
+        goto out;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, afr_ftruncate_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->ftruncate,
-                                           local->fd, local->cont.ftruncate.offset);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
+    gf_smsg(this->name, op_ret ? GF_LOG_ERROR : GF_LOG_INFO,
+            op_ret ? op_errno : 0, AFR_MSG_SET_PEND_XATTR, "name=%s",
+            priv->children[i]->name, "op_ret=%s",
+            op_ret ? "failed" : "succeeded", NULL);
 
-        return 0;
+out:
+    syncbarrier_wake(&local->barrier);
+    return 0;
 }
 
-
 int
-afr_ftruncate_done (call_frame_t *frame, xlator_t *this)
+afr_emptyb_set_pending_changelog(call_frame_t *frame, xlator_t *this,
+                                 unsigned char *locked_nodes)
 {
-        afr_local_t *local = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int ret = 0, i = 0;
 
-        local = frame->local;
+    local = frame->local;
+    priv = this->private;
 
-        local->transaction.unwind (frame, this);
+    AFR_ONLIST(locked_nodes, frame, afr_emptyb_set_pending_changelog_cbk,
+               xattrop, &local->loc, GF_XATTROP_ADD_ARRAY, local->xattr_req,
+               NULL);
 
-        AFR_STACK_DESTROY (frame);
+    /* It is sufficient if xattrop was successful on one child */
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid)
+            continue;
 
-        return 0;
+        if (local->replies[i].op_ret == 0) {
+            ret = 0;
+            goto out;
+        } else {
+            ret = afr_higher_errno(ret, local->replies[i].op_errno);
+        }
+    }
+out:
+    return -ret;
 }
 
+static int
+_afr_handle_empty_brick_type(xlator_t *this, call_frame_t *frame, loc_t *loc,
+                             int empty_index, afr_transaction_type type,
+                             char *op_type, const int op_type_len)
+{
+    int count = 0;
+    int ret = -ENOMEM;
+    int idx = -1;
+    int d_idx = -1;
+    unsigned char *locked_nodes = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+    local = frame->local;
+
+    locked_nodes = alloca0(priv->child_count);
+
+    idx = afr_index_for_transaction_type(type);
+    d_idx = afr_index_for_transaction_type(AFR_DATA_TRANSACTION);
+
+    local->pending = afr_matrix_create(priv->child_count, AFR_NUM_CHANGE_LOGS);
+    if (!local->pending)
+        goto out;
+
+    local->pending[empty_index][idx] = hton32(1);
+
+    if ((priv->esh_granular) && (type == AFR_ENTRY_TRANSACTION))
+        local->pending[empty_index][d_idx] = hton32(1);
+
+    local->xdata_req = dict_new();
+    if (!local->xdata_req)
+        goto out;
+
+    ret = dict_set_nstrn(local->xdata_req, "replicate-brick-op",
+                         SLEN("replicate-brick-op"), op_type, op_type_len);
+    if (ret)
+        goto out;
+
+    local->xattr_req = dict_new();
+    if (!local->xattr_req)
+        goto out;
+
+    ret = afr_set_pending_dict(priv, local->xattr_req, local->pending);
+    if (ret < 0)
+        goto out;
+
+    if (AFR_ENTRY_TRANSACTION == type) {
+        count = afr_selfheal_entrylk(frame, this, loc->inode, this->name, NULL,
+                                     locked_nodes);
+    } else {
+        count = afr_selfheal_inodelk(frame, this, loc->inode, this->name,
+                                     LLONG_MAX - 1, 0, locked_nodes);
+    }
+
+    if (!count) {
+        gf_smsg(this->name, GF_LOG_ERROR, EAGAIN, AFR_MSG_REPLACE_BRICK_STATUS,
+                NULL);
+        ret = -EAGAIN;
+        goto unlock;
+    }
+
+    ret = afr_emptyb_set_pending_changelog(frame, this, locked_nodes);
+    if (ret)
+        goto unlock;
+    ret = 0;
+unlock:
+    if (AFR_ENTRY_TRANSACTION == type) {
+        afr_selfheal_unentrylk(frame, this, loc->inode, this->name, NULL,
+                               locked_nodes, NULL);
+    } else {
+        afr_selfheal_uninodelk(frame, this, loc->inode, this->name,
+                               LLONG_MAX - 1, 0, locked_nodes);
+    }
+out:
+    return ret;
+}
 
-int
-afr_do_ftruncate (call_frame_t *frame, xlator_t *this)
+void
+afr_brick_args_cleanup(void *opaque)
 {
-        call_frame_t * transaction_frame = NULL;
-        afr_local_t *  local             = NULL;
+    afr_empty_brick_args_t *data = NULL;
 
-        int op_ret   = -1;
-        int op_errno = 0;
+    data = opaque;
+    loc_wipe(&data->loc);
+    GF_FREE(data);
+}
 
-        local = frame->local;
+int
+_afr_handle_empty_brick_cbk(int ret, call_frame_t *frame, void *opaque)
+{
+    afr_brick_args_cleanup(opaque);
+    return 0;
+}
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory.");
-                goto out;
-        }
+int
+_afr_handle_empty_brick(void *opaque)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int empty_index = -1;
+    int ret = -1;
+    int op_errno = ENOMEM;
+    call_frame_t *frame = NULL;
+    xlator_t *this = NULL;
+    char *op_type = NULL;
+    int op_type_len = 0;
+    afr_empty_brick_args_t *data = NULL;
+    call_frame_t *op_frame = NULL;
+
+    data = opaque;
+    frame = data->frame;
+    empty_index = data->empty_index;
+    if (!data->op_type)
+        goto out;
+
+    op_frame = copy_frame(frame);
+    if (!op_frame) {
+        ret = -1;
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    op_type = data->op_type;
+    op_type_len = strlen(op_type);
+    this = op_frame->this;
+    priv = this->private;
+
+    afr_set_lk_owner(op_frame, this, op_frame->root);
+    local = AFR_FRAME_INIT(op_frame, op_errno);
+    if (!local)
+        goto out;
+
+    loc_copy(&local->loc, &data->loc);
+
+    gf_smsg(this->name, GF_LOG_INFO, 0, AFR_MSG_NEW_BRICK, "name=%s",
+            priv->children[empty_index]->name, NULL);
+
+    ret = _afr_handle_empty_brick_type(this, op_frame, &local->loc, empty_index,
+                                       AFR_METADATA_TRANSACTION, op_type,
+                                       op_type_len);
+    if (ret) {
+        op_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+
+    dict_unref(local->xdata_req);
+    dict_unref(local->xattr_req);
+    afr_matrix_cleanup(local->pending, priv->child_count);
+    local->pending = NULL;
+    local->xattr_req = NULL;
+    local->xdata_req = NULL;
+
+    ret = _afr_handle_empty_brick_type(this, op_frame, &local->loc, empty_index,
+                                       AFR_ENTRY_TRANSACTION, op_type,
+                                       op_type_len);
+    if (ret) {
+        op_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+    ret = 0;
+out:
+    if (op_frame) {
+        AFR_STACK_DESTROY(op_frame);
+    }
+    AFR_STACK_UNWIND(setxattr, frame, ret, op_errno, NULL);
+    return 0;
+}
 
-        transaction_frame->local = local;
-        frame->local = NULL;
+int
+afr_split_brain_resolve_do(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                           char *data)
+{
+    afr_local_t *local = NULL;
+    int ret = -1;
+    int op_errno = EINVAL;
+
+    local = frame->local;
+    local->xdata_req = dict_new();
+
+    if (!local->xdata_req) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    ret = dict_set_int32_sizen(local->xdata_req, "heal-op",
+                               GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK);
+    if (ret) {
+        op_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+    ret = dict_set_str_sizen(local->xdata_req, "child-name", data);
+    if (ret) {
+        op_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+    /* set spb choice to -1 whether heal succeeds or not:
+     * If heal succeeds : spb-choice should be set to -1 as
+     *                    it is no longer valid; file is not
+     *                    in split-brain anymore.
+     * If heal doesn't succeed:
+     *                    spb-choice should be set to -1
+     *                    otherwise reads will be served
+     *                    from spb-choice which is misleading.
+     */
+    ret = afr_inode_split_brain_choice_set(loc->inode, this, -1);
+    if (ret)
+        gf_smsg(this->name, GF_LOG_WARNING, 0, AFR_MSG_SPLIT_BRAIN_SET_FAILED,
+                NULL);
+    afr_heal_splitbrain_file(frame, this, loc);
+    ret = 0;
+out:
+    if (ret < 0)
+        AFR_STACK_UNWIND(setxattr, frame, -1, op_errno, NULL);
+    return 0;
+}
 
-        local->op = GF_FOP_FTRUNCATE;
+int
+afr_get_split_brain_child_index(xlator_t *this, void *value, size_t len)
+{
+    int spb_child_index = -1;
+    char *spb_child_str = NULL;
 
-        local->transaction.fop    = afr_ftruncate_wind;
-        local->transaction.done   = afr_ftruncate_done;
-        local->transaction.unwind = afr_ftruncate_unwind;
+    spb_child_str = alloca0(len + 1);
+    memcpy(spb_child_str, value, len);
 
-        local->transaction.main_frame = frame;
+    if (!strcmp(spb_child_str, "none"))
+        return -2;
 
-        local->transaction.start   = 0;
-        local->transaction.len     = local->cont.ftruncate.offset;
+    spb_child_index = afr_get_child_index_from_name(this, spb_child_str);
+    if (spb_child_index < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, AFR_MSG_INVALID_SUBVOL,
+                "subvol=%s", spb_child_str, NULL);
+    }
+    return spb_child_index;
+}
 
-        afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
+int
+afr_can_set_split_brain_choice(void *opaque)
+{
+    afr_spbc_timeout_t *data = opaque;
+    call_frame_t *frame = NULL;
+    xlator_t *this = NULL;
+    loc_t *loc = NULL;
+    int ret = -1;
+
+    frame = data->frame;
+    loc = data->loc;
+    this = frame->this;
+
+    ret = afr_is_split_brain(frame, this, loc->inode, loc->gfid, &data->d_spb,
+                             &data->m_spb);
+
+    if (ret)
+        gf_smsg(this->name, GF_LOG_ERROR, 0,
+                AFR_MSG_SPLIT_BRAIN_DETERMINE_FAILED, "gfid=%s",
+                uuid_utoa(loc->gfid), NULL);
+    return ret;
+}
 
-        op_ret = 0;
+int
+afr_handle_split_brain_commands(xlator_t *this, call_frame_t *frame, loc_t *loc,
+                                dict_t *dict)
+{
+    void *choice_value = NULL;
+    void *resolve_value = NULL;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_spbc_timeout_t *data = NULL;
+    int len = 0;
+    int spb_child_index = -1;
+    int ret = -1;
+    int op_errno = EINVAL;
+
+    priv = this->private;
+
+    ret = dict_get_ptr_and_len(dict, GF_AFR_SBRAIN_CHOICE, &choice_value, &len);
+    ret = dict_get_ptr_and_len(dict, GF_AFR_SBRAIN_RESOLVE, &resolve_value,
+                               &len);
+    if (!choice_value && !resolve_value) {
+        ret = -1;
+        goto out;
+    }
+
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local) {
+        ret = 1;
+        goto out;
+    }
+
+    local->op = GF_FOP_SETXATTR;
+
+    if (choice_value) {
+        spb_child_index = afr_get_split_brain_child_index(this, choice_value,
+                                                          len);
+        if (spb_child_index < 0) {
+            /* Case where value was "none" */
+            if (spb_child_index == -2)
+                spb_child_index = -1;
+            else {
+                ret = 1;
+                op_errno = EINVAL;
+                goto out;
+            }
+        }
+
+        data = GF_CALLOC(1, sizeof(*data), gf_afr_mt_spbc_timeout_t);
+        if (!data) {
+            ret = 1;
+            goto out;
+        }
+        data->spb_child_index = spb_child_index;
+        data->frame = frame;
+        loc_copy(&local->loc, loc);
+        data->loc = &local->loc;
+        ret = synctask_new(this->ctx->env, afr_can_set_split_brain_choice,
+                           afr_set_split_brain_choice, NULL, data);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN_STATUS,
+                    "name=%s", loc->name, NULL);
+            ret = 1;
+            op_errno = ENOMEM;
+            goto out;
+        }
+        ret = 0;
+        goto out;
+    }
+
+    if (resolve_value) {
+        spb_child_index = afr_get_split_brain_child_index(this, resolve_value,
+                                                          len);
+        if (spb_child_index < 0) {
+            ret = 1;
+            goto out;
+        }
+
+        afr_split_brain_resolve_do(frame, this, loc,
+                                   priv->children[spb_child_index]->name);
+        ret = 0;
+    }
 out:
-        if (op_ret == -1) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (ftruncate, frame, op_ret, op_errno, NULL, NULL);
-        }
-
-        return 0;
+    /* key was correct but value was invalid when ret == 1 */
+    if (ret == 1) {
+        AFR_STACK_UNWIND(setxattr, frame, -1, op_errno, NULL);
+        if (data)
+            GF_FREE(data);
+        ret = 0;
+    }
+    return ret;
 }
 
-
 int
-afr_ftruncate (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, off_t offset)
+afr_handle_spb_choice_timeout(xlator_t *this, call_frame_t *frame, dict_t *dict)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t   *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = 0;
+    uint64_t timeout = 0;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
 
-        int ret = -1;
+    ret = dict_get_uint64(dict, GF_AFR_SPB_CHOICE_TIMEOUT, &timeout);
+    if (!ret) {
+        priv->spb_choice_timeout = timeout * 60;
+        AFR_STACK_UNWIND(setxattr, frame, ret, op_errno, NULL);
+    }
 
-        int op_ret   = -1;
-        int op_errno = 0;
+    return ret;
+}
 
-        uint64_t ctx;
-        afr_fd_ctx_t *fd_ctx = NULL;
+int
+afr_handle_empty_brick(xlator_t *this, call_frame_t *frame, loc_t *loc,
+                       dict_t *dict)
+{
+    int ret = -1;
+    int ab_ret = -1;
+    int empty_index = -1;
+    int op_errno = EPERM;
+    char *empty_brick = NULL;
+    char *op_type = NULL;
+    afr_empty_brick_args_t *data = NULL;
+
+    ret = dict_get_str_sizen(dict, GF_AFR_REPLACE_BRICK, &empty_brick);
+    if (!ret)
+        op_type = GF_AFR_REPLACE_BRICK;
+
+    ab_ret = dict_get_str_sizen(dict, GF_AFR_ADD_BRICK, &empty_brick);
+    if (!ab_ret)
+        op_type = GF_AFR_ADD_BRICK;
+
+    if (ret && ab_ret)
+        goto out;
+
+    if (frame->root->pid != GF_CLIENT_PID_ADD_REPLICA_MOUNT) {
+        gf_smsg(this->name, GF_LOG_ERROR, EPERM, AFR_MSG_INTERNAL_ATTR,
+                "op_type=%s", op_type, NULL);
+        ret = 1;
+        goto out;
+    }
+    empty_index = afr_get_child_index_from_name(this, empty_brick);
+
+    if (empty_index < 0) {
+        /* Didn't belong to this replica pair
+         * Just do a no-op
+         */
+        AFR_STACK_UNWIND(setxattr, frame, 0, 0, NULL);
+        return 0;
+    } else {
+        data = GF_CALLOC(1, sizeof(*data), gf_afr_mt_empty_brick_t);
+        if (!data) {
+            ret = 1;
+            op_errno = ENOMEM;
+            goto out;
+        }
+        data->frame = frame;
+        loc_copy(&data->loc, loc);
+        data->empty_index = empty_index;
+        data->op_type = op_type;
+        ret = synctask_new(this->ctx->env, _afr_handle_empty_brick,
+                           _afr_handle_empty_brick_cbk, NULL, data);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN_STATUS,
+                    NULL);
+            ret = 1;
+            op_errno = ENOMEM;
+            afr_brick_args_cleanup(data);
+            goto out;
+        }
+    }
+    ret = 0;
+out:
+    if (ret == 1) {
+        AFR_STACK_UNWIND(setxattr, frame, -1, op_errno, NULL);
+        ret = 0;
+    }
+    return ret;
+}
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+static int
+afr_handle_special_xattr(xlator_t *this, call_frame_t *frame, loc_t *loc,
+                         dict_t *dict)
+{
+    int ret = -1;
 
-        priv = this->private;
+    ret = afr_handle_split_brain_commands(this, frame, loc, dict);
+    if (ret == 0)
+        goto out;
 
-        ALLOC_OR_GOTO (local, afr_local_t, out);
+    ret = afr_handle_spb_choice_timeout(this, frame, dict);
+    if (ret == 0)
+        goto out;
 
-        ret = AFR_LOCAL_INIT (local, priv);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
-        }
+    /* Applicable for replace-brick and add-brick commands */
+    ret = afr_handle_empty_brick(this, frame, loc, dict);
+out:
+    return ret;
+}
 
-        frame->local = local;
+int
+afr_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+             int32_t flags, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = EINVAL;
 
-        local->cont.ftruncate.offset  = offset;
-        local->cont.ftruncate.ino     = fd->inode->ino;
+    GF_IF_INTERNAL_XATTR_GOTO("trusted.afr.*", dict, op_errno, out);
 
-        local->fd = fd_ref (fd);
+    GF_IF_INTERNAL_XATTR_GOTO("trusted.glusterfs.afr.*", dict, op_errno, out);
 
-        ret = fd_ctx_get (fd, this, &ctx);
-        if (ret < 0) {
-                goto out;
-        }
+    ret = afr_handle_special_xattr(this, frame, loc, dict);
+    if (ret == 0)
+        return 0;
 
-        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-        if (fd_ctx->up_count < priv->up_count) {
-                local->openfd_flush_cbk = afr_do_ftruncate;
-                afr_openfd_flush (frame, this, fd);
-        } else {
-                afr_do_ftruncate (frame, this);
-        }
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        op_ret = 0;
-out:
-        if (op_ret == -1) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (ftruncate, frame, op_ret, op_errno, NULL, NULL);
-        }
+    local->cont.setxattr.dict = dict_ref(dict);
+    local->cont.setxattr.flags = flags;
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
 
-        return 0;
-}
+    if (!local->xdata_req)
+        goto out;
 
-/* }}} */
+    local->transaction.wind = afr_setxattr_wind;
+    local->transaction.unwind = afr_setxattr_unwind;
 
-/* {{{ setattr */
+    loc_copy(&local->loc, loc);
+    ret = afr_set_inode_local(this, local, loc->inode);
+    if (ret)
+        goto out;
 
-int
-afr_setattr_unwind (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
+    local->transaction.main_frame = frame;
+    local->transaction.start = LLONG_MAX - 1;
+    local->transaction.len = 0;
 
-        local = frame->local;
+    local->op = GF_FOP_SETXATTR;
 
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame)
-                        main_frame = local->transaction.main_frame;
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
+    ret = afr_transaction(transaction_frame, this, AFR_METADATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
 
-        if (main_frame) {
-                local->cont.setattr.preop_buf.ia_ino  = local->cont.setattr.ino;
-                local->cont.setattr.postop_buf.ia_ino = local->cont.setattr.ino;
+    return 0;
+out:
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-                AFR_STACK_UNWIND (setattr, main_frame, local->op_ret,
-                                  local->op_errno,
-                                  &local->cont.setattr.preop_buf,
-                                  &local->cont.setattr.postop_buf);
-        }
+    AFR_STACK_UNWIND(setxattr, frame, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
+/* {{{ fsetxattr */
 
 int
-afr_setattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno,
-                      struct iatt *preop, struct iatt *postop)
+afr_fsetxattr_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
-
-        int child_index = (long) cookie;
-        int read_child  = 0;
-        int call_count  = -1;
-        int need_unwind = 0;
-
-        local = frame->local;
-        priv  = this->private;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-        read_child = afr_read_child (this, local->loc.inode);
+    local = frame->local;
 
-        LOCK (&frame->lock);
-        {
-                if (child_index == read_child) {
-                        local->read_child_returned = _gf_true;
-                }
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
+        return 0;
 
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
+    AFR_STACK_UNWIND(fsetxattr, main_frame, local->op_ret, local->op_errno,
+                     local->xdata_rsp);
+    return 0;
+}
 
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                                local->cont.setattr.preop_buf  = *preop;
-                                local->cont.setattr.postop_buf = *postop;
-                        }
+int
+afr_fsetxattr_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, NULL,
+                                 NULL, NULL, xdata);
+}
 
-                        if (child_index == read_child) {
-                                local->cont.setattr.preop_buf  = *preop;
-                                local->cont.setattr.postop_buf = *postop;
-                        }
+int
+afr_fsetxattr_wind(call_frame_t *frame, xlator_t *this, int subvol)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    STACK_WIND_COOKIE(frame, afr_fsetxattr_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->fsetxattr, local->fd,
+                      local->cont.fsetxattr.dict, local->cont.fsetxattr.flags,
+                      local->xdata_req);
+    return 0;
+}
 
-                        local->success_count++;
+int
+afr_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+              int32_t flags, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
 
-                        if ((local->success_count >= priv->wait_count)
-                            && local->read_child_returned) {
-                                need_unwind = 1;
-                        }
-                }
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
+    GF_IF_INTERNAL_XATTR_GOTO("trusted.afr.*", dict, op_errno, out);
 
-        if (need_unwind)
-                local->transaction.unwind (frame, this);
+    GF_IF_INTERNAL_XATTR_GOTO("trusted.glusterfs.afr.*", dict, op_errno, out);
 
-        call_count = afr_frame_return (frame);
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-        }
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        return 0;
-}
+    local->cont.fsetxattr.dict = dict_ref(dict);
+    local->cont.fsetxattr.flags = flags;
 
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
 
-int32_t
-afr_setattr_wind (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    if (!local->xdata_req)
+        goto out;
 
-        int call_count = -1;
-        int i = 0;
+    local->transaction.wind = afr_fsetxattr_wind;
+    local->transaction.unwind = afr_fsetxattr_unwind;
 
-        local = frame->local;
-        priv = this->private;
+    local->fd = fd_ref(fd);
+    ret = afr_set_inode_local(this, local, fd->inode);
+    if (ret)
+        goto out;
 
-        call_count = afr_up_children_count (priv->child_count, local->child_up);
+    local->op = GF_FOP_FSETXATTR;
 
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
+    local->transaction.main_frame = frame;
+    local->transaction.start = LLONG_MAX - 1;
+    local->transaction.len = 0;
 
-        local->call_count = call_count;
+    ret = afr_transaction(transaction_frame, this, AFR_METADATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, afr_setattr_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->setattr,
-                                           &local->loc,
-                                           &local->cont.setattr.in_buf,
-                                           local->cont.setattr.valid);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
+    return 0;
+out:
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-        return 0;
+    AFR_STACK_UNWIND(fsetxattr, frame, -1, op_errno, NULL);
+    return 0;
 }
 
+/* }}} */
+
+/* {{{ removexattr */
 
 int
-afr_setattr_done (call_frame_t *frame, xlator_t *this)
+afr_removexattr_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *local = NULL;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-        local = frame->local;
-
-        local->transaction.unwind (frame, this);
-
-        AFR_STACK_DESTROY (frame);
+    local = frame->local;
 
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
+
+    AFR_STACK_UNWIND(removexattr, main_frame, local->op_ret, local->op_errno,
+                     local->xdata_rsp);
+    return 0;
 }
 
+int
+afr_removexattr_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, NULL,
+                                 NULL, NULL, xdata);
+}
 
 int
-afr_setattr (call_frame_t *frame, xlator_t *this,
-             loc_t *loc, struct iatt *buf, int32_t valid)
+afr_removexattr_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t   *transaction_frame = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        int ret = -1;
+    local = frame->local;
+    priv = this->private;
 
-        int op_ret   = -1;
-        int op_errno = 0;
+    STACK_WIND_COOKIE(frame, afr_removexattr_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->removexattr, &local->loc,
+                      local->cont.removexattr.name, local->xdata_req);
+    return 0;
+}
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+int
+afr_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                const char *name, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
 
-        priv = this->private;
+    GF_IF_NATIVE_XATTR_GOTO("trusted.afr.*", name, op_errno, out);
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory.");
-                goto out;
-        }
+    GF_IF_NATIVE_XATTR_GOTO("trusted.glusterfs.afr.*", name, op_errno, out);
 
-        ALLOC_OR_GOTO (local, afr_local_t, out);
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-        ret = AFR_LOCAL_INIT (local, priv);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
-        }
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        transaction_frame->local = local;
+    local->cont.removexattr.name = gf_strdup(name);
 
-        local->op_ret = -1;
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
 
-        local->cont.setattr.ino     = loc->inode->ino;
+    if (!local->xdata_req)
+        goto out;
 
-        local->cont.setattr.in_buf = *buf;
-        local->cont.setattr.valid  = valid;
+    local->transaction.wind = afr_removexattr_wind;
+    local->transaction.unwind = afr_removexattr_unwind;
 
-        local->transaction.fop    = afr_setattr_wind;
-        local->transaction.done   = afr_setattr_done;
-        local->transaction.unwind = afr_setattr_unwind;
+    loc_copy(&local->loc, loc);
+    ret = afr_set_inode_local(this, local, loc->inode);
+    if (ret)
+        goto out;
 
-        loc_copy (&local->loc, loc);
+    local->op = GF_FOP_REMOVEXATTR;
 
-        local->transaction.main_frame = frame;
-        local->transaction.start   = LLONG_MAX - 1;
-        local->transaction.len     = 0;
+    local->transaction.main_frame = frame;
+    local->transaction.start = LLONG_MAX - 1;
+    local->transaction.len = 0;
 
-        afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+    ret = afr_transaction(transaction_frame, this, AFR_METADATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
 
-        op_ret = 0;
+    return 0;
 out:
-        if (op_ret == -1) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (setattr, frame, op_ret, op_errno, NULL, NULL);
-        }
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-        return 0;
+    AFR_STACK_UNWIND(removexattr, frame, -1, op_errno, NULL);
+    return 0;
 }
 
-/* {{{ fsetattr */
-
+/* ffremovexattr */
 int
-afr_fsetattr_unwind (call_frame_t *frame, xlator_t *this)
+afr_fremovexattr_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame)
-                        main_frame = local->transaction.main_frame;
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
-
-        if (main_frame) {
-                local->cont.fsetattr.preop_buf.ia_ino  =
-                        local->cont.fsetattr.ino;
-                local->cont.fsetattr.postop_buf.ia_ino =
-                        local->cont.fsetattr.ino;
-
-                AFR_STACK_UNWIND (fsetattr, main_frame, local->op_ret,
-                                  local->op_errno,
-                                  &local->cont.fsetattr.preop_buf,
-                                  &local->cont.fsetattr.postop_buf);
-        }
+    local = frame->local;
 
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(fremovexattr, main_frame, local->op_ret, local->op_errno,
+                     local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_fsetattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno,
-                      struct iatt *preop, struct iatt *postop)
+afr_fremovexattr_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
-
-        int child_index = (long) cookie;
-        int read_child  = 0;
-        int call_count  = -1;
-        int need_unwind = 0;
-
-        local = frame->local;
-        priv  = this->private;
-
-        read_child = afr_read_child (this, local->fd->inode);
-
-        LOCK (&frame->lock);
-        {
-                if (child_index == read_child) {
-                        local->read_child_returned = _gf_true;
-                }
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, NULL,
+                                 NULL, NULL, xdata);
+}
 
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
+int
+afr_fremovexattr_wind(call_frame_t *frame, xlator_t *this, int subvol)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                                local->cont.fsetattr.preop_buf  = *preop;
-                                local->cont.fsetattr.postop_buf = *postop;
-                        }
+    local = frame->local;
+    priv = this->private;
 
-                        if (child_index == read_child) {
-                                local->cont.fsetattr.preop_buf  = *preop;
-                                local->cont.fsetattr.postop_buf = *postop;
-                        }
+    STACK_WIND_COOKIE(frame, afr_fremovexattr_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->fremovexattr, local->fd,
+                      local->cont.removexattr.name, local->xdata_req);
+    return 0;
+}
 
-                        local->success_count++;
+int
+afr_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                 const char *name, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
 
-                        if ((local->success_count >= priv->wait_count)
-                            && local->read_child_returned) {
-                                need_unwind = 1;
-                        }
-                }
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
+    GF_IF_NATIVE_XATTR_GOTO("trusted.afr.*", name, op_errno, out);
 
-        if (need_unwind)
-                local->transaction.unwind (frame, this);
+    GF_IF_NATIVE_XATTR_GOTO("trusted.glusterfs.afr.*", name, op_errno, out);
 
-        call_count = afr_frame_return (frame);
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-        }
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        return 0;
-}
+    local->cont.removexattr.name = gf_strdup(name);
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
 
+    if (!local->xdata_req)
+        goto out;
 
-int32_t
-afr_fsetattr_wind (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    local->transaction.wind = afr_fremovexattr_wind;
+    local->transaction.unwind = afr_fremovexattr_unwind;
 
-        int call_count = -1;
-        int i = 0;
+    local->fd = fd_ref(fd);
+    ret = afr_set_inode_local(this, local, fd->inode);
+    if (ret)
+        goto out;
 
-        local = frame->local;
-        priv = this->private;
+    local->op = GF_FOP_FREMOVEXATTR;
 
-        call_count = afr_up_children_count (priv->child_count, local->child_up);
+    local->transaction.main_frame = frame;
+    local->transaction.start = LLONG_MAX - 1;
+    local->transaction.len = 0;
 
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
+    ret = afr_transaction(transaction_frame, this, AFR_METADATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
 
-        local->call_count = call_count;
+    return 0;
+out:
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, afr_fsetattr_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->fsetattr,
-                                           local->fd,
-                                           &local->cont.fsetattr.in_buf,
-                                           local->cont.fsetattr.valid);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
+    AFR_STACK_UNWIND(fremovexattr, frame, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int
-afr_fsetattr_done (call_frame_t *frame, xlator_t *this)
+afr_fallocate_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *local = NULL;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-        local = frame->local;
-
-        local->transaction.unwind (frame, this);
-
-        AFR_STACK_DESTROY (frame);
+    local = frame->local;
 
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(fallocate, main_frame, local->op_ret, local->op_errno,
+                     &local->cont.inode_wfop.prebuf,
+                     &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_fsetattr (call_frame_t *frame, xlator_t *this,
-              fd_t *fd, struct iatt *buf, int32_t valid)
+afr_fallocate_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                       struct iatt *postbuf, dict_t *xdata)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t   *transaction_frame = NULL;
-
-        int ret = -1;
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, prebuf,
+                                 postbuf, NULL, xdata);
+}
 
-        int op_ret   = -1;
-        int op_errno = 0;
+int
+afr_fallocate_wind(call_frame_t *frame, xlator_t *this, int subvol)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    STACK_WIND_COOKIE(frame, afr_fallocate_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->fallocate, local->fd,
+                      local->cont.fallocate.mode, local->cont.fallocate.offset,
+                      local->cont.fallocate.len, local->xdata_req);
+    return 0;
+}
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+int
+afr_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+              off_t offset, size_t len, dict_t *xdata)
+{
+    call_frame_t *transaction_frame = NULL;
+    afr_local_t *local = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
 
-        priv = this->private;
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory.");
-                goto out;
-        }
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        ALLOC_OR_GOTO (local, afr_local_t, out);
+    local->cont.fallocate.mode = mode;
+    local->cont.fallocate.offset = offset;
+    local->cont.fallocate.len = len;
 
-        ret = AFR_LOCAL_INIT (local, priv);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
-        }
+    local->fd = fd_ref(fd);
+    ret = afr_set_inode_local(this, local, fd->inode);
+    if (ret)
+        goto out;
 
-        transaction_frame->local = local;
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
 
-        local->op_ret = -1;
+    if (!local->xdata_req)
+        goto out;
 
-        local->cont.fsetattr.ino     = fd->inode->ino;
+    local->op = GF_FOP_FALLOCATE;
 
-        local->cont.fsetattr.in_buf = *buf;
-        local->cont.fsetattr.valid  = valid;
+    local->transaction.wind = afr_fallocate_wind;
+    local->transaction.unwind = afr_fallocate_unwind;
 
-        local->transaction.fop    = afr_fsetattr_wind;
-        local->transaction.done   = afr_fsetattr_done;
-        local->transaction.unwind = afr_fsetattr_unwind;
+    local->transaction.main_frame = frame;
 
-        local->fd                 = fd_ref (fd);
+    local->transaction.start = local->cont.fallocate.offset;
+    local->transaction.len = 0;
 
-        local->transaction.main_frame = frame;
-        local->transaction.start   = LLONG_MAX - 1;
-        local->transaction.len     = 0;
+    afr_fix_open(fd, this);
 
-        afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+    ret = afr_transaction(transaction_frame, this, AFR_DATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
 
-        op_ret = 0;
+    return 0;
 out:
-        if (op_ret == -1) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (fsetattr, frame, op_ret, op_errno, NULL, NULL);
-        }
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-        return 0;
+    AFR_STACK_UNWIND(fallocate, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
+/* }}} */
 
-/* {{{ setxattr */
-
+/* {{{ discard */
 
 int
-afr_setxattr_unwind (call_frame_t *frame, xlator_t *this)
+afr_discard_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
-
-        local = frame->local;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame)
-                        main_frame = local->transaction.main_frame;
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
+    local = frame->local;
 
-        if (main_frame) {
-                AFR_STACK_UNWIND (setxattr, main_frame,
-                                  local->op_ret, local->op_errno)
-        }
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(discard, main_frame, local->op_ret, local->op_errno,
+                     &local->cont.inode_wfop.prebuf,
+                     &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_setxattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno)
+afr_discard_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
 {
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
-
-        int call_count  = -1;
-        int need_unwind = 0;
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, prebuf,
+                                 postbuf, NULL, xdata);
+}
 
-        local = frame->local;
-        priv = this->private;
+int
+afr_discard_wind(call_frame_t *frame, xlator_t *this, int subvol)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    STACK_WIND_COOKIE(frame, afr_discard_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->discard, local->fd,
+                      local->cont.discard.offset, local->cont.discard.len,
+                      local->xdata_req);
+    return 0;
+}
 
-        LOCK (&frame->lock);
-        {
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                        }
-                        local->success_count++;
+int
+afr_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+            size_t len, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
 
-                        if (local->success_count == priv->child_count) {
-                                need_unwind = 1;
-                        }
-                }
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        if (need_unwind)
-                local->transaction.unwind (frame, this);
+    local->cont.discard.offset = offset;
+    local->cont.discard.len = len;
 
-        call_count = afr_frame_return (frame);
+    local->fd = fd_ref(fd);
+    ret = afr_set_inode_local(this, local, fd->inode);
+    if (ret)
+        goto out;
 
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-        }
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
 
-        return 0;
-}
+    if (!local->xdata_req)
+        goto out;
 
+    local->op = GF_FOP_DISCARD;
 
-int
-afr_setxattr_wind (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    local->transaction.wind = afr_discard_wind;
+    local->transaction.unwind = afr_discard_unwind;
 
-        int call_count = -1;
-        int i = 0;
+    local->transaction.main_frame = frame;
 
-        local = frame->local;
-        priv = this->private;
+    local->transaction.start = local->cont.discard.offset;
+    local->transaction.len = 0;
 
-        call_count = afr_up_children_count (priv->child_count, local->child_up);
+    afr_fix_open(fd, this);
 
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
+    ret = afr_transaction(transaction_frame, this, AFR_DATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
 
-        local->call_count = call_count;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, afr_setxattr_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->setxattr,
-                                           &local->loc,
-                                           local->cont.setxattr.dict,
-                                           local->cont.setxattr.flags);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
+    return 0;
+out:
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-        return 0;
+    AFR_STACK_UNWIND(discard, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
+/* {{{ zerofill */
 
 int
-afr_setxattr_done (call_frame_t *frame, xlator_t *this)
+afr_zerofill_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t * local = frame->local;
-
-        local->transaction.unwind (frame, this);
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-        AFR_STACK_DESTROY (frame);
+    local = frame->local;
 
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
+
+    AFR_STACK_UNWIND(discard, main_frame, local->op_ret, local->op_errno,
+                     &local->cont.inode_wfop.prebuf,
+                     &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+    return 0;
 }
 
 int
-afr_setxattr (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, dict_t *dict, int32_t flags)
+afr_zerofill_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xdata)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t   *transaction_frame = NULL;
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, prebuf,
+                                 postbuf, NULL, xdata);
+}
 
-        int ret = -1;
+int
+afr_zerofill_wind(call_frame_t *frame, xlator_t *this, int subvol)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    STACK_WIND_COOKIE(frame, afr_zerofill_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->zerofill, local->fd,
+                      local->cont.zerofill.offset, local->cont.zerofill.len,
+                      local->xdata_req);
+    return 0;
+}
 
-        int op_ret   = -1;
-        int op_errno = 0;
+int
+afr_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             size_t len, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-        priv = this->private;
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        ALLOC_OR_GOTO (local, afr_local_t, out);
+    local->cont.zerofill.offset = offset;
+    local->cont.zerofill.len = len;
 
-        ret = AFR_LOCAL_INIT (local, priv);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
-        }
+    local->fd = fd_ref(fd);
+    ret = afr_set_inode_local(this, local, fd->inode);
+    if (ret)
+        goto out;
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory.");
-                goto out;
-        }
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
 
-        transaction_frame->local = local;
+    if (!local->xdata_req)
+        goto out;
 
-        local->op_ret = -1;
+    local->op = GF_FOP_ZEROFILL;
 
-        local->cont.setxattr.dict  = dict_ref (dict);
-        local->cont.setxattr.flags = flags;
+    local->transaction.wind = afr_zerofill_wind;
+    local->transaction.unwind = afr_zerofill_unwind;
 
-        local->transaction.fop    = afr_setxattr_wind;
-        local->transaction.done   = afr_setxattr_done;
-        local->transaction.unwind = afr_setxattr_unwind;
+    local->transaction.main_frame = frame;
 
-        loc_copy (&local->loc, loc);
+    local->transaction.start = local->cont.zerofill.offset;
+    local->transaction.len = len;
 
-        local->transaction.main_frame = frame;
-        local->transaction.start   = LLONG_MAX - 1;
-        local->transaction.len     = 0;
+    afr_fix_open(fd, this);
 
-        afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+    ret = afr_transaction(transaction_frame, this, AFR_DATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
 
-        op_ret = 0;
+    return 0;
 out:
-        if (op_ret == -1) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (setxattr, frame, op_ret, op_errno);
-        }
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-        return 0;
+    AFR_STACK_UNWIND(zerofill, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
 /* }}} */
 
-/* {{{ removexattr */
+int32_t
+afr_xattrop_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xattr,
+                     dict_t *xdata)
+{
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, NULL,
+                                 NULL, xattr, xdata);
+}
 
+int
+afr_xattrop_wind(call_frame_t *frame, xlator_t *this, int subvol)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    STACK_WIND_COOKIE(frame, afr_xattrop_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->xattrop, &local->loc,
+                      local->cont.xattrop.optype, local->cont.xattrop.xattr,
+                      local->xdata_req);
+    return 0;
+}
 
 int
-afr_removexattr_unwind (call_frame_t *frame, xlator_t *this)
+afr_xattrop_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame)
-                        main_frame = local->transaction.main_frame;
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
-
-        if (main_frame) {
-                AFR_STACK_UNWIND (removexattr, main_frame,
-                                  local->op_ret, local->op_errno)
-        }
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(xattrop, main_frame, local->op_ret, local->op_errno,
+                     local->xattr_rsp, local->xdata_rsp);
+    return 0;
+}
 
-int
-afr_removexattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno)
+int32_t
+afr_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc,
+            gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
 {
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
 
-        int call_count  = -1;
-        int need_unwind = 0;
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-        local = frame->local;
-        priv = this->private;
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        LOCK (&frame->lock);
-        {
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                        }
-                        local->success_count++;
+    local->cont.xattrop.xattr = dict_ref(xattr);
+    local->cont.xattrop.optype = optype;
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
 
-                        if (local->success_count == priv->wait_count) {
-                                need_unwind = 1;
-                        }
-                }
+    local->transaction.wind = afr_xattrop_wind;
+    local->transaction.unwind = afr_xattrop_unwind;
 
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
+    loc_copy(&local->loc, loc);
+    ret = afr_set_inode_local(this, local, loc->inode);
+    if (ret)
+        goto out;
 
-        if (need_unwind)
-                local->transaction.unwind (frame, this);
+    local->op = GF_FOP_XATTROP;
 
-        call_count = afr_frame_return (frame);
+    local->transaction.main_frame = frame;
+    local->transaction.start = LLONG_MAX - 1;
+    local->transaction.len = 0;
 
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-        }
+    ret = afr_transaction(transaction_frame, this, AFR_METADATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
 
-        return 0;
-}
+    return 0;
+out:
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
+    AFR_STACK_UNWIND(xattrop, frame, -1, op_errno, NULL, NULL);
+    return 0;
+}
 
 int32_t
-afr_removexattr_wind (call_frame_t *frame, xlator_t *this)
+afr_fxattrop_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xattr,
+                      dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-
-        int call_count = -1;
-        int i = 0;
-
-        local = frame->local;
-        priv = this->private;
-
-        call_count = afr_up_children_count (priv->child_count, local->child_up);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
-
-        local->call_count = call_count;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, afr_removexattr_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->removexattr,
-                                           &local->loc,
-                                           local->cont.removexattr.name);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
-
-        return 0;
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, NULL,
+                                 NULL, xattr, xdata);
 }
 
-
 int
-afr_removexattr_done (call_frame_t *frame, xlator_t *this)
+afr_fxattrop_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t * local = frame->local;
-
-        local->transaction.unwind (frame, this);
-
-        AFR_STACK_DESTROY (frame);
-
-        return 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    STACK_WIND_COOKIE(frame, afr_fxattrop_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->fxattrop, local->fd,
+                      local->cont.xattrop.optype, local->cont.xattrop.xattr,
+                      local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_removexattr (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc, const char *name)
+afr_fxattrop_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t   *transaction_frame = NULL;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-        int ret = -1;
+    local = frame->local;
 
-        int op_ret   = -1;
-        int op_errno = 0;
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
+        return 0;
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
-        VALIDATE_OR_GOTO (loc, out);
+    AFR_STACK_UNWIND(fxattrop, main_frame, local->op_ret, local->op_errno,
+                     local->xattr_rsp, local->xdata_rsp);
+    return 0;
+}
 
-        priv = this->private;
+int32_t
+afr_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
+             gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
+
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
+
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
+
+    local->cont.xattrop.xattr = dict_ref(xattr);
+    local->cont.xattrop.optype = optype;
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
+
+    local->transaction.wind = afr_fxattrop_wind;
+    local->transaction.unwind = afr_fxattrop_unwind;
+
+    local->fd = fd_ref(fd);
+    ret = afr_set_inode_local(this, local, fd->inode);
+    if (ret)
+        goto out;
+
+    local->op = GF_FOP_FXATTROP;
+
+    local->transaction.main_frame = frame;
+    local->transaction.start = LLONG_MAX - 1;
+    local->transaction.len = 0;
+
+    ret = afr_transaction(transaction_frame, this, AFR_METADATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
+out:
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory.");
-                goto out;
-        }
+    AFR_STACK_UNWIND(fxattrop, frame, -1, op_errno, NULL, NULL);
+    return 0;
+}
 
-        ALLOC_OR_GOTO (local, afr_local_t, out);
+int
+afr_fsync_unwind(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-        ret = AFR_LOCAL_INIT (local, priv);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
-        }
+    local = frame->local;
 
-        transaction_frame->local = local;
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
+        return 0;
 
-        local->op_ret = -1;
+    AFR_STACK_UNWIND(fsync, main_frame, local->op_ret, local->op_errno,
+                     &local->cont.inode_wfop.prebuf,
+                     &local->cont.inode_wfop.postbuf, local->xdata_rsp);
 
-        local->cont.removexattr.name = gf_strdup (name);
+    return 0;
+}
 
-        local->transaction.fop    = afr_removexattr_wind;
-        local->transaction.done   = afr_removexattr_done;
-        local->transaction.unwind = afr_removexattr_unwind;
+int
+afr_fsync_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                   struct iatt *postbuf, dict_t *xdata)
+{
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, prebuf,
+                                 postbuf, NULL, xdata);
+}
 
-        loc_copy (&local->loc, loc);
+int
+afr_fsync_wind(call_frame_t *frame, xlator_t *this, int subvol)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        local->transaction.main_frame = frame;
-        local->transaction.start   = LLONG_MAX - 1;
-        local->transaction.len     = 0;
+    local = frame->local;
+    priv = this->private;
 
-        afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+    STACK_WIND_COOKIE(frame, afr_fsync_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->fsync, local->fd,
+                      local->cont.fsync.datasync, local->xdata_req);
+    return 0;
+}
 
-        op_ret = 0;
+int
+afr_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+          dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int32_t op_errno = ENOMEM;
+    int8_t last_fsync = 0;
+
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
+
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
+
+    if (xdata) {
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+        if (dict_get_int8(xdata, "last-fsync", &last_fsync) == 0) {
+            if (last_fsync) {
+                local->transaction.disable_delayed_post_op = _gf_true;
+            }
+        }
+    } else {
+        local->xdata_req = dict_new();
+    }
+
+    if (!local->xdata_req)
+        goto out;
+
+    local->fd = fd_ref(fd);
+    ret = afr_set_inode_local(this, local, fd->inode);
+    if (ret)
+        goto out;
+
+    local->op = GF_FOP_FSYNC;
+    local->cont.fsync.datasync = datasync;
+
+    if (afr_fd_has_witnessed_unstable_write(this, fd->inode)) {
+        /* don't care. we only wanted to CLEAR the bit */
+    }
+
+    local->transaction.wind = afr_fsync_wind;
+    local->transaction.unwind = afr_fsync_unwind;
+
+    local->transaction.main_frame = frame;
+
+    ret = afr_transaction(transaction_frame, this, AFR_DATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
 out:
-        if (op_ret == -1) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (removexattr, frame, op_ret, op_errno);
-        }
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-        return 0;
+    AFR_STACK_UNWIND(fsync, frame, -1, op_errno, NULL, NULL, NULL);
+
+    return 0;
 }
diff --git a/xlators/cluster/afr/src/afr-inode-write.h b/xlators/cluster/afr/src/afr-inode-write.h
index 47589872206..a787069b7a1 100644
--- a/xlators/cluster/afr/src/afr-inode-write.h
+++ b/xlators/cluster/afr/src/afr-inode-write.h
@@ -1,72 +1,94 @@
 /*
-   Copyright (c) 2007-2010 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __INODE_WRITE_H__
 #define __INODE_WRITE_H__
 
 int32_t
-afr_chmod (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, mode_t mode);
+afr_chmod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          dict_t *xdata);
 
 int32_t
-afr_chown (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, uid_t uid, gid_t gid);
+afr_chown(call_frame_t *frame, xlator_t *this, loc_t *loc, uid_t uid, gid_t gid,
+          dict_t *xdata);
 
 int
-afr_fchown (call_frame_t *frame, xlator_t *this,
-	    fd_t *fd, uid_t uid, gid_t gid);
+afr_fchown(call_frame_t *frame, xlator_t *this, fd_t *fd, uid_t uid, gid_t gid,
+           dict_t *xdata);
+
+int32_t
+afr_fchmod(call_frame_t *frame, xlator_t *this, fd_t *fd, mode_t mode,
+           dict_t *xdata);
+
+int32_t
+afr_writev(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
+           int32_t count, off_t offset, uint32_t flags, struct iobref *iobref,
+           dict_t *xdata);
+
+int32_t
+afr_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+             dict_t *xdata);
+
+int32_t
+afr_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+              dict_t *xdata);
 
 int32_t
-afr_fchmod (call_frame_t *frame, xlator_t *this,
-	    fd_t *fd, mode_t mode);
+afr_utimens(call_frame_t *frame, xlator_t *this, loc_t *loc,
+            struct timespec tv[2], dict_t *xdata);
+
+int
+afr_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *buf,
+            int32_t valid, dict_t *xdata);
+
+int
+afr_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *buf,
+             int32_t valid, dict_t *xdata);
 
 int32_t
-afr_writev (call_frame_t *frame, xlator_t *this, fd_t *fd, 
-	    struct iovec *vector, int32_t count, off_t offset,
-            struct iobref *iobref);
+afr_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+             int32_t flags, dict_t *xdata);
 
 int32_t
-afr_truncate (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, off_t offset);
+afr_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+              int32_t flags, dict_t *xdata);
 
 int32_t
-afr_ftruncate (call_frame_t *frame, xlator_t *this,
-	       fd_t *fd, off_t offset);
+afr_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                const char *name, dict_t *xdata);
 
 int32_t
-afr_utimens (call_frame_t *frame, xlator_t *this,
-	     loc_t *loc, struct timespec tv[2]);
+afr_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                 const char *name, dict_t *xdata);
+
+int
+afr_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+            size_t len, dict_t *xdata);
 
 int
-afr_setattr (call_frame_t *frame, xlator_t *this,
-             loc_t *loc, struct iatt *buf, int32_t valid);
+afr_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+              off_t offset, size_t len, dict_t *xdata);
 
 int
-afr_fsetattr (call_frame_t *frame, xlator_t *this,
-              fd_t *fd, struct iatt *buf, int32_t valid);
+afr_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             off_t len, dict_t *xdata);
 
 int32_t
-afr_setxattr (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, dict_t *dict, int32_t flags);
+afr_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc,
+            gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata);
 
 int32_t
-afr_removexattr (call_frame_t *frame, xlator_t *this,
-		 loc_t *loc, const char *name);
+afr_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
+             gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata);
 
+int
+afr_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+          dict_t *xdata);
 #endif /* __INODE_WRITE_H__ */
diff --git a/xlators/cluster/afr/src/afr-lk-common.c b/xlators/cluster/afr/src/afr-lk-common.c
index d2f7579fc42..bc8eabe0f43 100644
--- a/xlators/cluster/afr/src/afr-lk-common.c
+++ b/xlators/cluster/afr/src/afr-lk-common.c
@@ -1,2183 +1,791 @@
 /*
-  Copyright (c) 2007-2010 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
-#include "dict.h"
-#include "byte-order.h"
-#include "common-utils.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/byte-order.h>
+#include <glusterfs/common-utils.h>
 
 #include "afr.h"
 #include "afr-transaction.h"
+#include "afr-messages.h"
 
 #include <signal.h>
 
-
-#define LOCKED_NO       0x0        /* no lock held */
-#define LOCKED_YES      0x1        /* for DATA, METADATA, ENTRY and higher_path */
-#define LOCKED_LOWER    0x2        /* for lower path */
-
-int
-afr_lock_blocking (call_frame_t *frame, xlator_t *this, int child_index);
-
-static uint64_t afr_lock_number = 1;
-
-static uint64_t
-get_afr_lock_number ()
-{
-        return (++afr_lock_number);
-}
-
-int
-afr_set_lock_number (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t         *local    = NULL;
-        afr_internal_lock_t *int_lock = NULL;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        int_lock->lock_number = get_afr_lock_number ();
-
-        return 0;
-}
+#define LOCKED_NO 0x0    /* no lock held */
+#define LOCKED_YES 0x1   /* for DATA, METADATA, ENTRY and higher_path */
+#define LOCKED_LOWER 0x2 /* for lower path */
 
 void
-afr_set_lk_owner (call_frame_t *frame, xlator_t *this)
-{
-        if (!frame->root->lk_owner) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "Setting lk-owner=%llu",
-                        (unsigned long long) (unsigned long)frame->root);
-                frame->root->lk_owner = (uint64_t) (unsigned long)frame->root;
-        }
-}
-
-static int
-is_afr_lock_selfheal (afr_local_t *local)
+afr_lockee_cleanup(afr_lockee_t *lockee)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        int                  ret      = -1;
-
-        int_lock = &local->internal_lock;
-
-        switch (int_lock->selfheal_lk_type) {
-        case AFR_DATA_SELF_HEAL_LK:
-        case AFR_METADATA_SELF_HEAL_LK:
-                ret = 1;
-                break;
-        case AFR_ENTRY_SELF_HEAL_LK:
-                ret = 0;
-                break;
-        }
+    if (lockee->fd) {
+        fd_unref(lockee->fd);
+        lockee->fd = NULL;
+    } else {
+        loc_wipe(&lockee->loc);
+    }
 
-        return ret;
+    GF_FREE(lockee->basename);
+    lockee->basename = NULL;
+    GF_FREE(lockee->locked_nodes);
+    lockee->locked_nodes = NULL;
 
-}
-
-int32_t
-internal_lock_count (call_frame_t *frame, xlator_t *this,
-                     afr_fd_ctx_t *fd_ctx)
-{
-        afr_local_t   *local = NULL;
-        afr_private_t *priv  = NULL;
-
-        int32_t call_count = 0;
-        int i = 0;
-
-        local = frame->local;
-        priv  = this->private;
-
-        if (fd_ctx) {
-                GF_ASSERT (local->fd);
-                for (i = 0; i < priv->child_count; i++) {
-                        if (local->child_up[i] && fd_ctx->opened_on[i])
-                                ++call_count;
-                }
-        } else {
-                for (i = 0; i < priv->child_count; i++) {
-                        if (local->child_up[i])
-                                ++call_count;
-                }
-        }
-
-        return call_count;
-}
-
-static void
-afr_print_inodelk (char *str, int size, int cmd,
-                   struct gf_flock *flock, uint64_t owner)
-{
-        char *cmd_str = NULL;
-        char *type_str = NULL;
-
-        switch (cmd) {
-#if F_GETLK != F_GETLK64
-        case F_GETLK64:
-#endif
-        case F_GETLK:
-                cmd_str = "GETLK";
-                break;
-
-#if F_SETLK != F_SETLK64
-        case F_SETLK64:
-#endif
-        case F_SETLK:
-                cmd_str = "SETLK";
-                break;
-
-#if F_SETLKW != F_SETLKW64
-        case F_SETLKW64:
-#endif
-        case F_SETLKW:
-                cmd_str = "SETLKW";
-                break;
-
-        default:
-                cmd_str = "<null>";
-                break;
-        }
-
-        switch (flock->l_type) {
-        case F_RDLCK:
-                type_str = "READ";
-                break;
-        case F_WRLCK:
-                type_str = "WRITE";
-                break;
-        case F_UNLCK:
-                type_str = "UNLOCK";
-                break;
-        default:
-                type_str = "UNKNOWN";
-                break;
-        }
-
-        snprintf (str, size, "lock=INODELK, cmd=%s, type=%s, "
-                  "start=%llu, len=%llu, pid=%llu, lk-owner=%llu",
-                  cmd_str, type_str, (unsigned long long) flock->l_start,
-                  (unsigned long long) flock->l_len,
-                  (unsigned long long) flock->l_pid,
-                  (unsigned long long) owner);
-
-}
-
-static void
-afr_print_lockee (char *str, int size, loc_t *loc, fd_t *fd,
-                  int child_index)
-{
-        snprintf (str, size, "path=%s, fd=%p, child=%d",
-                  loc->path ? loc->path : "<nul>",
-                  fd ? fd : NULL,
-                  child_index);
+    return;
 }
 
 void
-afr_print_entrylk (char *str, int size, const char *basename,
-                   uint64_t owner)
+afr_lockees_cleanup(afr_internal_lock_t *int_lock)
 {
-        snprintf (str, size, "Basename=%s, lk-owner=%llu",
-                  basename ? basename : "<nul>",
-                  (unsigned long long)owner);
-}
+    int i = 0;
 
-static void
-afr_print_verdict (int op_ret, int op_errno, char *str)
-{
-        if (op_ret < 0) {
-                if (op_errno == EAGAIN)
-                        strcpy (str, "EAGAIN");
-                else
-                        strcpy (str, "FAILED");
-        }
-        else
-                strcpy (str, "GRANTED");
-}
-
-static void
-afr_set_lock_call_type (afr_lock_call_type_t lock_call_type,
-                        char *lock_call_type_str,
-                        afr_internal_lock_t *int_lock)
-{
-        switch (lock_call_type) {
-        case AFR_INODELK_TRANSACTION:
-                if (int_lock->transaction_lk_type == AFR_TRANSACTION_LK)
-                        strcpy (lock_call_type_str, "AFR_INODELK_TRANSACTION");
-                else
-                        strcpy (lock_call_type_str, "AFR_INODELK_SELFHEAL");
-                break;
-        case AFR_INODELK_NB_TRANSACTION:
-                if (int_lock->transaction_lk_type == AFR_TRANSACTION_LK)
-                        strcpy (lock_call_type_str, "AFR_INODELK_NB_TRANSACTION");
-                else
-                        strcpy (lock_call_type_str, "AFR_INODELK_NB_SELFHEAL");
-                break;
-        case AFR_ENTRYLK_TRANSACTION:
-                if (int_lock->transaction_lk_type == AFR_TRANSACTION_LK)
-                        strcpy (lock_call_type_str, "AFR_ENTRYLK_TRANSACTION");
-                else
-                        strcpy (lock_call_type_str, "AFR_ENTRYLK_SELFHEAL");
-                break;
-        case AFR_ENTRYLK_NB_TRANSACTION:
-                if (int_lock->transaction_lk_type == AFR_TRANSACTION_LK)
-                        strcpy (lock_call_type_str, "AFR_ENTRYLK_NB_TRANSACTION");
-                else
-                        strcpy (lock_call_type_str, "AFR_ENTRYLK_NB_SELFHEAL");
-                break;
-        default:
-                strcpy (lock_call_type_str, "UNKNOWN");
-                break;
-        }
-
-}
-
-static void
-afr_trace_inodelk_out (call_frame_t *frame, afr_lock_call_type_t lock_call_type,
-                       afr_lock_op_type_t lk_op_type, struct gf_flock *flock,
-                       int op_ret, int op_errno, int32_t child_index)
-{
-        xlator_t            *this     = NULL;
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_private_t       *priv     = NULL;
-
-        char lockee[256];
-        char lock_call_type_str[256];
-        char verdict[16];
-
-        this     = THIS;
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        priv     = this->private;
-
-        if (!priv->inodelk_trace) {
-                return;
-        }
-
-        afr_print_lockee (lockee, 256, &local->loc, local->fd, child_index);
-
-        afr_set_lock_call_type (lock_call_type, lock_call_type_str, int_lock);
-
-        afr_print_verdict (op_ret, op_errno, verdict);
-
-        gf_log (this->name, GF_LOG_NORMAL,
-                "[%s %s] [%s] Lockee={%s} Number={%llu}",
-                lock_call_type_str,
-                lk_op_type == AFR_LOCK_OP ? "LOCK REPLY" : "UNLOCK REPLY",
-                verdict,
-                lockee,
-                (unsigned long long) int_lock->lock_number);
+    for (i = 0; i < int_lock->lockee_count; i++) {
+        afr_lockee_cleanup(&int_lock->lockee[i]);
+    }
 
+    return;
 }
-
-static void
-afr_trace_inodelk_in (call_frame_t *frame, afr_lock_call_type_t lock_call_type,
-                      afr_lock_op_type_t lk_op_type, struct gf_flock *flock,
-                      int32_t cmd, int32_t child_index)
-{
-        xlator_t            *this     = NULL;
-        afr_local_t         *local    = NULL;
-        afr_internal_lock_t *int_lock = NULL;
-        afr_private_t       *priv     = NULL;
-
-        char lock[256];
-        char lockee[256];
-        char lock_call_type_str[256];
-
-        this     = THIS;
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        priv     = this->private;
-
-        if (!priv->inodelk_trace) {
-                return;
-        }
-
-        afr_print_inodelk (lock, 256, cmd, flock, frame->root->lk_owner);
-        afr_print_lockee (lockee, 256, &local->loc, local->fd, child_index);
-
-        afr_set_lock_call_type (lock_call_type, lock_call_type_str, int_lock);
-
-        gf_log (this->name, GF_LOG_NORMAL,
-                "[%s %s] Lock={%s} Lockee={%s} Number={%llu}",
-                lock_call_type_str,
-                lk_op_type == AFR_LOCK_OP ? "LOCK REQUEST" : "UNLOCK REQUEST",
-                lock, lockee,
-                (unsigned long long) int_lock->lock_number);
-
+int
+afr_entry_lockee_cmp(const void *l1, const void *l2)
+{
+    const afr_lockee_t *r1 = l1;
+    const afr_lockee_t *r2 = l2;
+    int ret = 0;
+    uuid_t gfid1 = {0};
+    uuid_t gfid2 = {0};
+
+    loc_gfid((loc_t *)&r1->loc, gfid1);
+    loc_gfid((loc_t *)&r2->loc, gfid2);
+    ret = gf_uuid_compare(gfid1, gfid2);
+    /*Entrylks with NULL basename are the 'smallest'*/
+    if (ret == 0) {
+        if (!r1->basename)
+            return -1;
+        if (!r2->basename)
+            return 1;
+        ret = strcmp(r1->basename, r2->basename);
+    }
+
+    if (ret <= 0)
+        return -1;
+    else
+        return 1;
 }
 
-static void
-afr_trace_entrylk_in (call_frame_t *frame, afr_lock_call_type_t lock_call_type,
-                      afr_lock_op_type_t lk_op_type, const char *basename,
-                      int32_t child_index)
-{
-        xlator_t            *this     = NULL;
-        afr_local_t         *local    = NULL;
-        afr_internal_lock_t *int_lock = NULL;
-        afr_private_t       *priv     = NULL;
-
-        char lock[256];
-        char lockee[256];
-        char lock_call_type_str[256];
-
-        this     = THIS;
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        priv     = this->private;
-
-        if (!priv->entrylk_trace) {
-                return;
-        }
-
-        afr_print_entrylk (lock, 256, basename, frame->root->lk_owner);
-        afr_print_lockee (lockee, 256, &local->loc, local->fd, child_index);
-
-        afr_set_lock_call_type (lock_call_type, lock_call_type_str, int_lock);
-
-        gf_log (this->name, GF_LOG_NORMAL,
-                "[%s %s] Lock={%s} Lockee={%s} Number={%llu}",
-                lock_call_type_str,
-                lk_op_type == AFR_LOCK_OP ? "LOCK REQUEST" : "UNLOCK REQUEST",
-                lock, lockee,
-                (unsigned long long) int_lock->lock_number);
-}
+int
+afr_lock_blocking(call_frame_t *frame, xlator_t *this, int child_index);
 
-static void
-afr_trace_entrylk_out (call_frame_t *frame, afr_lock_call_type_t lock_call_type,
-                       afr_lock_op_type_t lk_op_type, const char *basename, int op_ret,
-                       int op_errno, int32_t child_index)
+void
+afr_set_lk_owner(call_frame_t *frame, xlator_t *this, void *lk_owner)
 {
-        xlator_t            *this     = NULL;
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_private_t       *priv     = NULL;
-
-        char lock[256];
-        char lockee[256];
-        char lock_call_type_str[256];
-        char verdict[16];
-
-        this     = THIS;
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        priv     = this->private;
-
-        if (!priv->entrylk_trace) {
-                return;
-        }
-
-        afr_print_lockee (lockee, 256, &local->loc, local->fd, child_index);
-
-        afr_set_lock_call_type (lock_call_type, lock_call_type_str, int_lock);
-
-        afr_print_verdict (op_ret, op_errno, verdict);
-
-        gf_log (this->name, GF_LOG_NORMAL,
-                "[%s %s] [%s] Lock={%s} Lockee={%s} Number={%llu}",
-                lock_call_type_str,
-                lk_op_type == AFR_LOCK_OP ? "LOCK REPLY" : "UNLOCK REPLY",
-                verdict,
-                lock, lockee,
-                (unsigned long long) int_lock->lock_number);
+    gf_msg_trace(this->name, 0, "Setting lk-owner=%llu",
+                 (unsigned long long)(unsigned long)lk_owner);
 
+    set_lk_owner_from_ptr(&frame->root->lk_owner, lk_owner);
 }
 
-static int
-transaction_lk_op (afr_local_t *local)
+int32_t
+internal_lock_count(call_frame_t *frame, xlator_t *this)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        int ret = -1;
-
-        int_lock = &local->internal_lock;
-
-        if (int_lock->transaction_lk_type == AFR_TRANSACTION_LK) {
-                gf_log (THIS->name, GF_LOG_DEBUG,
-                        "lk op is for a transaction");
-                ret = 1;
-        }
-        else if (int_lock->transaction_lk_type == AFR_SELFHEAL_LK) {
-                gf_log (THIS->name, GF_LOG_DEBUG,
-                        "lk op is for a self heal");
-
-                ret = 0;
-        }
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int32_t call_count = 0;
+    int i = 0;
 
-        if (ret == -1)
-                gf_log (THIS->name, GF_LOG_DEBUG,
-                        "lk op is not set");
+    local = frame->local;
+    priv = this->private;
 
-        return ret;
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->child_up[i])
+            ++call_count;
+    }
 
+    return call_count;
 }
 
-static int
-is_afr_lock_transaction (afr_local_t *local)
+int
+afr_add_entry_lockee(afr_local_t *local, loc_t *loc, char *basename,
+                     int child_count)
 {
-        int ret = 0;
+    int ret = -ENOMEM;
+    afr_internal_lock_t *int_lock = &local->internal_lock;
+    afr_lockee_t *lockee = &int_lock->lockee[int_lock->lockee_count];
 
-        switch (local->transaction.type) {
-	case AFR_DATA_TRANSACTION:
-	case AFR_METADATA_TRANSACTION:
-                ret = 1;
-                break;
+    GF_ASSERT(int_lock->lockee_count < AFR_LOCKEE_COUNT_MAX);
+    loc_copy(&lockee->loc, loc);
+    lockee->basename = (basename) ? gf_strdup(basename) : NULL;
+    if (basename && !lockee->basename)
+        goto out;
 
-        case AFR_ENTRY_RENAME_TRANSACTION:
-        case AFR_ENTRY_TRANSACTION:
-                ret = 0;
-                break;
+    lockee->locked_count = 0;
+    lockee->locked_nodes = GF_CALLOC(child_count, sizeof(*lockee->locked_nodes),
+                                     gf_afr_mt_afr_node_character);
 
-        }
+    if (!lockee->locked_nodes)
+        goto out;
 
-        return ret;
+    ret = 0;
+    int_lock->lockee_count++;
+out:
+    if (ret) {
+        afr_lockee_cleanup(lockee);
+    }
+    return ret;
 }
 
-static int
-initialize_entrylk_variables (call_frame_t *frame, xlator_t *this)
+int
+afr_add_inode_lockee(afr_local_t *local, int child_count)
 {
-        afr_local_t         *local    = NULL;
-        afr_internal_lock_t *int_lock = NULL;
-        afr_private_t       *priv     = NULL;
+    int ret = -ENOMEM;
+    afr_internal_lock_t *int_lock = &local->internal_lock;
+    afr_lockee_t *lockee = &int_lock->lockee[int_lock->lockee_count];
 
-        int i = 0;
+    if (local->fd) {
+        lockee->fd = fd_ref(local->fd);
+    } else {
+        loc_copy(&lockee->loc, &local->loc);
+    }
 
-        priv     = this->private;
-        local    = frame->local;
-        int_lock = &local->internal_lock;
+    lockee->locked_count = 0;
+    lockee->locked_nodes = GF_CALLOC(child_count, sizeof(*lockee->locked_nodes),
+                                     gf_afr_mt_afr_node_character);
 
-        int_lock->entrylk_lock_count = 0;
-        int_lock->lock_op_ret        = -1;
-        int_lock->lock_op_errno      = 0;
+    if (!lockee->locked_nodes)
+        goto out;
 
-        for (i = 0; i < priv->child_count; i++) {
-                int_lock->entry_locked_nodes[i] = 0;
-        }
-
-        return 0;
+    ret = 0;
+    int_lock->lockee_count++;
+out:
+    if (ret) {
+        afr_lockee_cleanup(lockee);
+    }
+    return ret;
 }
 
 static int
-initialize_inodelk_variables (call_frame_t *frame, xlator_t *this)
+initialize_internal_lock_variables(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t         *local    = NULL;
-        afr_internal_lock_t *int_lock = NULL;
-        afr_private_t       *priv     = NULL;
+    afr_local_t *local = NULL;
+    afr_internal_lock_t *int_lock = NULL;
+    afr_private_t *priv = NULL;
 
-        int i = 0;
+    int i = 0;
 
-        priv     = this->private;
-        local    = frame->local;
-        int_lock = &local->internal_lock;
+    priv = this->private;
+    local = frame->local;
+    int_lock = &local->internal_lock;
 
-        int_lock->inodelk_lock_count = 0;
-        int_lock->lock_op_ret        = -1;
-        int_lock->lock_op_errno      = 0;
+    int_lock->lock_count = 0;
+    int_lock->lock_op_ret = -1;
+    int_lock->lock_op_errno = 0;
+    int_lock->lk_attempted_count = 0;
 
-        for (i = 0; i < priv->child_count; i++) {
-                int_lock->inode_locked_nodes[i] = 0;
-        }
+    for (i = 0; i < AFR_LOCKEE_COUNT_MAX; i++) {
+        if (!int_lock->lockee[i].locked_nodes)
+            break;
+        int_lock->lockee[i].locked_count = 0;
+        memset(int_lock->lockee[i].locked_nodes, 0,
+               sizeof(*int_lock->lockee[i].locked_nodes) * priv->child_count);
+    }
 
-        return 0;
-}
-
-loc_t *
-lower_path (loc_t *l1, const char *b1, loc_t *l2, const char *b2)
-{
-	int ret = 0;
-
-	ret = strcmp (l1->path, l2->path);
-
-	if (ret == 0)
-		ret = strcmp (b1, b2);
-
-	if (ret <= 0)
-		return l1;
-	else
-		return l2;
+    return 0;
 }
 
 int
-afr_locked_nodes_count (unsigned char *locked_nodes, int child_count)
-
+afr_lockee_locked_nodes_count(afr_internal_lock_t *int_lock)
 {
-        int i;
-        int call_count = 0;
+    int call_count = 0;
+    int i = 0;
 
-        for (i = 0; i < child_count; i++) {
-                if (locked_nodes[i] & LOCKED_YES)
-                        call_count++;
-        }
+    for (i = 0; i < int_lock->lockee_count; i++)
+        call_count += int_lock->lockee[i].locked_count;
 
-        return call_count;
+    return call_count;
 }
 
-/* FIXME: What if UNLOCK fails */
-static int32_t
-afr_unlock_common_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno)
-{
-	afr_local_t         *local    = NULL;
-        afr_internal_lock_t *int_lock = NULL;
-	int call_count = 0;
-
-	local    = frame->local;
-        int_lock = &local->internal_lock;
-
-	LOCK (&frame->lock);
-	{
-		call_count = --int_lock->lk_call_count;
-	}
-	UNLOCK (&frame->lock);
-
-	if (call_count == 0) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "All internal locks unlocked");
-                int_lock->lock_cbk (frame, this);
-        }
-
-	return 0;
-}
+int
+afr_locked_nodes_count(unsigned char *locked_nodes, int child_count)
 
-static int32_t
-afr_unlock_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno)
 {
-        afr_trace_inodelk_out (frame, AFR_INODELK_TRANSACTION,
-                               AFR_UNLOCK_OP, NULL, op_ret,
-                               op_errno, (long) cookie);
-
-        if (op_ret < 0 && op_errno != ENOTCONN && op_errno != EBADFD) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "Unlock failed for some reason");
-        }
-
-        afr_unlock_common_cbk (frame, cookie, this, op_ret, op_errno);
+    int i = 0;
+    int call_count = 0;
 
-        return 0;
+    for (i = 0; i < child_count; i++) {
+        if (locked_nodes[i] & LOCKED_YES)
+            call_count++;
+    }
 
+    return call_count;
 }
 
-static int
-afr_unlock_inodelk (call_frame_t *frame, xlator_t *this)
+static void
+afr_log_locks_failure(call_frame_t *frame, char *where, char *what,
+                      int op_errno)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_private_t       *priv     = NULL;
-
-        struct gf_flock flock;
-        int call_count = 0;
-        int i = 0;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        priv     = this->private;
-
-        flock.l_start = int_lock->lk_flock.l_start;
-        flock.l_len   = int_lock->lk_flock.l_len;
-        flock.l_type  = F_UNLCK;
-
-        call_count = afr_locked_nodes_count (int_lock->inode_locked_nodes,
-                                             priv->child_count);
+    xlator_t *this = frame->this;
+    gf_lkowner_t *lk_owner = &frame->root->lk_owner;
+    afr_local_t *local = frame->local;
+    const char *fop = NULL;
+    char *gfid = NULL;
+    const char *name = NULL;
 
-        int_lock->lk_call_count = call_count;
+    fop = gf_fop_list[local->op];
 
-        if (!call_count) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "No internal locks unlocked");
-                int_lock->lock_cbk (frame, this);
-                goto out;
-        }
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (int_lock->inode_locked_nodes[i] & LOCKED_YES) {
-                        if (local->fd) {
-                                afr_trace_inodelk_in (frame, AFR_INODELK_TRANSACTION,
-                                                      AFR_UNLOCK_OP, &flock, F_SETLK, i);
-
-                                STACK_WIND_COOKIE (frame, afr_unlock_inodelk_cbk,
-                                                   (void *) (long)i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->finodelk,
-                                                   this->name, local->fd,
-                                                   F_SETLK, &flock);
-
-                                if (!--call_count)
-                                        break;
-
-                        } else {
-                                afr_trace_inodelk_in (frame, AFR_INODELK_TRANSACTION,
-                                                      AFR_UNLOCK_OP, &flock, F_SETLK, i);
-
-                                STACK_WIND_COOKIE (frame, afr_unlock_inodelk_cbk,
-                                                   (void *) (long)i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->inodelk,
-                                                   this->name, &local->loc,
-                                                   F_SETLK, &flock);
-
-                                if (!--call_count)
-                                        break;
-
-                        }
-
-                }
-
-        }
-
-out:
-        return 0;
+    switch (local->transaction.type) {
+        case AFR_ENTRY_RENAME_TRANSACTION:
+        case AFR_ENTRY_TRANSACTION:
+            switch (local->op) {
+                case GF_FOP_LINK:
+                    gfid = uuid_utoa(local->newloc.pargfid);
+                    name = local->newloc.name;
+                    break;
+                default:
+                    gfid = uuid_utoa(local->loc.pargfid);
+                    name = local->loc.name;
+                    break;
+            }
+            gf_msg(this->name, GF_LOG_WARNING, op_errno,
+                   AFR_MSG_INTERNAL_LKS_FAILED,
+                   "Unable to do entry %s with lk-owner:%s on %s "
+                   "while attempting %s on {pgfid:%s, name:%s}.",
+                   what, lkowner_utoa(lk_owner), where, fop, gfid, name);
+            break;
+        case AFR_DATA_TRANSACTION:
+        case AFR_METADATA_TRANSACTION:
+            gfid = uuid_utoa(local->inode->gfid);
+            gf_msg(this->name, GF_LOG_WARNING, op_errno,
+                   AFR_MSG_INTERNAL_LKS_FAILED,
+                   "Unable to do inode %s with lk-owner:%s on %s "
+                   "while attempting %s on gfid:%s.",
+                   what, lkowner_utoa(lk_owner), where, fop, gfid);
+            break;
+    }
 }
 
 static int32_t
-afr_unlock_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno)
+afr_unlock_common_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        afr_trace_entrylk_out (frame, AFR_ENTRYLK_TRANSACTION,
-                              AFR_UNLOCK_OP, NULL, op_ret,
-                               op_errno, (long) cookie);
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    afr_internal_lock_t *int_lock = NULL;
+    int lockee_num = 0;
+    int call_count = 0;
+    int child_index = 0;
+    int ret = 0;
 
-        afr_unlock_common_cbk (frame, cookie, this, op_ret, op_errno);
+    local = frame->local;
+    int_lock = &local->internal_lock;
+    priv = this->private;
+    lockee_num = (int)((long)cookie) / priv->child_count;
+    child_index = (int)((long)cookie) % priv->child_count;
 
-        return 0;
-}
-
-static int
-afr_unlock_entrylk (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_private_t       *priv     = NULL;
-        const char          *basename = NULL;
-        loc_t               *loc      = NULL;
-
-        int call_count = 0;
-        int i = -1;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        priv     = this->private;
-
-        basename = int_lock->lk_basename;
-        if (int_lock->lk_loc)
-                loc = int_lock->lk_loc;
-
-        call_count = afr_locked_nodes_count (int_lock->entry_locked_nodes,
-                                             priv->child_count);
-        int_lock->lk_call_count = call_count;
-
-        if (!call_count){
-                gf_log (this->name, GF_LOG_TRACE,
-                        "No internal locks unlocked");
-                int_lock->lock_cbk (frame, this);
-                goto out;
-        }
+    if (op_ret < 0 && op_errno != ENOTCONN && op_errno != EBADFD) {
+        afr_log_locks_failure(frame, priv->children[child_index]->name,
+                              "unlock", op_errno);
+    }
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (int_lock->entry_locked_nodes[i] & LOCKED_YES) {
-                        afr_trace_entrylk_in (frame, AFR_ENTRYLK_NB_TRANSACTION,
-                                              AFR_UNLOCK_OP, basename, i);
-
-                        STACK_WIND_COOKIE (frame, afr_unlock_entrylk_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->entrylk,
-                                           this->name,
-                                           loc, basename,
-                                           ENTRYLK_UNLOCK, ENTRYLK_WRLCK);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
-
-out:
-        return 0;
+    int_lock->lockee[lockee_num].locked_nodes[child_index] &= LOCKED_NO;
+    if (local->transaction.type == AFR_DATA_TRANSACTION && op_ret != 1)
+        ret = afr_write_subvol_reset(frame, this);
 
-}
+    LOCK(&frame->lock);
+    {
+        call_count = --int_lock->lk_call_count;
+    }
+    UNLOCK(&frame->lock);
 
-static int32_t
-afr_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-	      int32_t op_ret, int32_t op_errno)
-{
-        afr_internal_lock_t *int_lock = NULL;
-	afr_local_t         *local    = NULL;
-	afr_private_t       *priv     = NULL;
-
-	int done = 0;
-	int child_index = (long) cookie;
-
-	local    = frame->local;
-        int_lock = &local->internal_lock;
-	priv     = this->private;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret == -1) {
-			if (op_errno == ENOSYS) {
-				/* return ENOTSUP */
-				gf_log (this->name, GF_LOG_ERROR,
-					"subvolume does not support locking. "
-					"please load features/posix-locks xlator on server");
-				local->op_ret = op_ret;
-                                int_lock->lock_op_ret = op_ret;
-				done = 1;
-			}
-
-			local->child_up[child_index] = 0;
-			local->op_errno              = op_errno;
-                        int_lock->lock_op_errno      = op_errno;
-		}
-	}
-	UNLOCK (&frame->lock);
-
-        if ((op_ret == -1) &&
-            (op_errno == ENOSYS)) {
-                afr_unlock (frame, this);
-        } else {
-                if (op_ret == 0) {
-                        int_lock->locked_nodes[child_index]
-                                |= LOCKED_YES;
-                        int_lock->lock_count++;
-                }
-                afr_lock_blocking (frame, this, child_index + 1);
-        }
+    if (call_count == 0) {
+        int_lock->lock_cbk(frame, this);
+    }
 
-	return 0;
+    return ret;
 }
 
-static int32_t
-afr_blocking_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno)
-{
-        afr_trace_inodelk_out (frame, AFR_INODELK_TRANSACTION,
-                               AFR_LOCK_OP, NULL, op_ret,
-                               op_errno, (long) cookie);
-
-        afr_lock_cbk (frame, cookie, this, op_ret, op_errno);
-        return 0;
-
-}
-
-static int32_t
-afr_lock_lower_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno)
-{
-        afr_internal_lock_t *int_lock    = NULL;
-        afr_private_t       *priv        = NULL;
-        afr_local_t         *local       = NULL;
-        loc_t               *lower       = NULL;
-        loc_t               *higher      = NULL;
-        const char          *lower_name  = NULL;
-        const char          *higher_name = NULL;
-
-        int child_index = (long) cookie;
-
-        priv     = this->private;
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        if (op_errno == ENOSYS) {
-                                /* return ENOTSUP */
-
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "subvolume does not support locking. "
-                                        "please load features/posix-locks xlator on server");
-
-                                local->op_ret   = op_ret;
-                        }
-
-                        local->child_up[child_index] = 0;
-                        local->op_errno = op_errno;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if (op_ret != 0) {
-                afr_unlock (frame, this);
-                goto out;
-        } else {
-                int_lock->lower_locked_nodes[child_index] |= LOCKED_LOWER;
-                int_lock->lock_count++;
-        }
-
-        /* The lower path has been locked. Now lock the higher path */
-
-        lower = lower_path (&local->transaction.parent_loc,
-                            local->transaction.basename,
-                            &local->transaction.new_parent_loc,
-                            local->transaction.new_basename);
-
-        lower_name = (lower == &local->transaction.parent_loc ?
-                      local->transaction.basename :
-                      local->transaction.new_basename);
-
-        higher = (lower == &local->transaction.parent_loc ?
-                  &local->transaction.new_parent_loc :
-                  &local->transaction.parent_loc);
-
-        higher_name = (higher == &local->transaction.parent_loc ?
-                       local->transaction.basename :
-                       local->transaction.new_basename);
-
-        afr_trace_entrylk_in (frame, AFR_ENTRYLK_TRANSACTION,
-                              AFR_LOCK_OP, higher_name, child_index);
-
-
-        STACK_WIND_COOKIE (frame, afr_lock_cbk,
-                           (void *) (long) child_index,
-                           priv->children[child_index],
-                           priv->children[child_index]->fops->entrylk,
-                           this->name, higher, higher_name,
-                           ENTRYLK_LOCK, ENTRYLK_WRLCK);
-
-out:
-        return 0;
-}
-
-static int32_t
-afr_blocking_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno)
-{
-        afr_trace_entrylk_out (frame, AFR_ENTRYLK_TRANSACTION,
-                               AFR_LOCK_OP, NULL, op_ret,
-                               op_errno, (long)cookie);
-
-        afr_lock_cbk (frame, cookie, this, op_ret, op_errno);
-        return 0;
-}
-
-static int
-afr_copy_locked_nodes (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_private_t       *priv     = NULL;
-
-        priv     = this->private;
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        switch (local->transaction.type) {
-	case AFR_DATA_TRANSACTION:
-	case AFR_METADATA_TRANSACTION:
-                memcpy (int_lock->inode_locked_nodes,
-                        int_lock->locked_nodes,
-                        priv->child_count);
-                int_lock->inodelk_lock_count = int_lock->lock_count;
-                break;
-
-        case AFR_ENTRY_RENAME_TRANSACTION:
+void
+afr_internal_lock_wind(call_frame_t *frame,
+                       int32_t (*cbk)(call_frame_t *, void *, xlator_t *,
+                                      int32_t, int32_t, dict_t *),
+                       void *cookie, int child, int lockee_num,
+                       gf_boolean_t blocking, gf_boolean_t unlock)
+{
+    afr_local_t *local = frame->local;
+    xlator_t *this = frame->this;
+    afr_private_t *priv = this->private;
+    afr_internal_lock_t *int_lock = &local->internal_lock;
+    entrylk_cmd cmd = ENTRYLK_LOCK_NB;
+    int32_t cmd1 = F_SETLK;
+    struct gf_flock flock = {
+        0,
+    };
+
+    switch (local->transaction.type) {
         case AFR_ENTRY_TRANSACTION:
-                memcpy (int_lock->entry_locked_nodes,
-                        int_lock->locked_nodes,
-                        priv->child_count);
-                int_lock->entrylk_lock_count = int_lock->lock_count;
-                break;
-        }
-
-        return 0;
-
-}
-
-int
-afr_lock_blocking (call_frame_t *frame, xlator_t *this, int child_index)
-{
-        afr_internal_lock_t *int_lock    = NULL;
-        afr_local_t         *local       = NULL;
-	afr_private_t       *priv        = NULL;
-        afr_fd_ctx_t        *fd_ctx      = NULL;
-	loc_t               *lower       = NULL;
-	loc_t               *higher      = NULL;
-	const char          *lower_name  = NULL;
-	const char          *higher_name = NULL;
-
-	struct gf_flock flock;
-        uint64_t ctx;
-        int ret = 0;
-
-	local    = frame->local;
-        int_lock = &local->internal_lock;
-	priv     = this->private;
-
-	flock.l_start = int_lock->lk_flock.l_start;
-	flock.l_len   = int_lock->lk_flock.l_len;
-	flock.l_type  = int_lock->lk_flock.l_type;
-
-        if (local->fd) {
-                ret = fd_ctx_get (local->fd, this, &ctx);
-
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "unable to get fd ctx for fd=%p",
-                                local->fd);
-
-                        local->op_ret           = -1;
-                        int_lock->lock_op_ret   = -1;
-                        local->op_errno         = EINVAL;
-                        int_lock->lock_op_errno = EINVAL;
-
-                        afr_copy_locked_nodes (frame, this);
-
-                        afr_unlock (frame, this);
-
-                        return 0;
-                }
-
-                fd_ctx = (afr_fd_ctx_t *)(long) ctx;
-
-                /* skip over children that or down
-                   or don't have the fd open */
-
-                while ((child_index < priv->child_count)
-                       && (!local->child_up[child_index]
-                           || !fd_ctx->opened_on[child_index]))
-
-                        child_index++;
-        } else {
-                /* skip over children that are down */
-                while ((child_index < priv->child_count)
-                       && !local->child_up[child_index])
-                        child_index++;
-        }
-
-	if ((child_index == priv->child_count) &&
-	    int_lock->lock_count == 0) {
-
-		gf_log (this->name, GF_LOG_DEBUG,
-			"unable to lock on even one child");
-
-		local->op_ret           = -1;
-                int_lock->lock_op_ret   = -1;
-		local->op_errno         = EAGAIN;
-                int_lock->lock_op_errno = EAGAIN;
-
-                afr_copy_locked_nodes (frame, this);
-
-                afr_unlock(frame, this);
-
-		return 0;
-
-	}
-
-	if ((child_index == priv->child_count)
-	    || (int_lock->lock_count ==
-		afr_up_children_count (priv->child_count,
-                                       local->child_up))) {
-
-		/* we're done locking */
-
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "we're done locking");
-
-                afr_copy_locked_nodes (frame, this);
-
-                int_lock->lock_op_ret = 0;
-                int_lock->lock_cbk (frame, this);
-		return 0;
-	}
-
-	switch (local->transaction.type) {
-	case AFR_DATA_TRANSACTION:
-	case AFR_METADATA_TRANSACTION:
-
-		if (local->fd) {
-                        afr_trace_inodelk_in (frame, AFR_INODELK_TRANSACTION,
-                                              AFR_LOCK_OP, &flock, F_SETLKW,
-                                              child_index);
-
-			STACK_WIND_COOKIE (frame, afr_blocking_inodelk_cbk,
-					   (void *) (long) child_index,
-					   priv->children[child_index],
-					   priv->children[child_index]->fops->finodelk,
-					   this->name, local->fd,
-                                           F_SETLKW, &flock);
-
-		} else {
-                        afr_trace_inodelk_in (frame, AFR_INODELK_TRANSACTION,
-                                              AFR_LOCK_OP, &flock, F_SETLKW,
-                                              child_index);
-
-			STACK_WIND_COOKIE (frame, afr_blocking_inodelk_cbk,
-					   (void *) (long) child_index,
-					   priv->children[child_index],
-					   priv->children[child_index]->fops->inodelk,
-					   this->name, &local->loc,
-                                           F_SETLKW, &flock);
-		}
-
-		break;
-
-	case AFR_ENTRY_RENAME_TRANSACTION:
-	{
-		lower = lower_path (&local->transaction.parent_loc,
-				    local->transaction.basename,
-				    &local->transaction.new_parent_loc,
-				    local->transaction.new_basename);
-
-		lower_name = (lower == &local->transaction.parent_loc ?
-			      local->transaction.basename :
-			      local->transaction.new_basename);
-
-		higher = (lower == &local->transaction.parent_loc ?
-			  &local->transaction.new_parent_loc :
-			  &local->transaction.parent_loc);
-
-		higher_name = (higher == &local->transaction.parent_loc ?
-			       local->transaction.basename :
-			       local->transaction.new_basename);
-
-                afr_trace_entrylk_in (frame, AFR_ENTRYLK_TRANSACTION,
-                                      AFR_LOCK_OP, lower_name, child_index);
-
-
-		STACK_WIND_COOKIE (frame, afr_lock_lower_cbk,
-				   (void *) (long) child_index,
-				   priv->children[child_index],
-				   priv->children[child_index]->fops->entrylk,
-				   this->name, lower, lower_name,
-				   ENTRYLK_LOCK, ENTRYLK_WRLCK);
-
-		break;
-	}
-
-	case AFR_ENTRY_TRANSACTION:
-		if (local->fd) {
-                        afr_trace_entrylk_in (frame, AFR_ENTRYLK_TRANSACTION,
-                                              AFR_LOCK_OP, local->transaction.basename,
-                                              child_index);
-
-			STACK_WIND_COOKIE (frame, afr_blocking_entrylk_cbk,
-					   (void *) (long) child_index,
-					   priv->children[child_index],
-					   priv->children[child_index]->fops->fentrylk,
-					   this->name, local->fd,
-					   local->transaction.basename,
-					   ENTRYLK_LOCK, ENTRYLK_WRLCK);
-		} else {
-                        afr_trace_entrylk_in (frame, AFR_ENTRYLK_TRANSACTION,
-                                              AFR_LOCK_OP, local->transaction.basename,
-                                              child_index);
-
-			STACK_WIND_COOKIE (frame, afr_blocking_entrylk_cbk,
-					   (void *) (long) child_index,
-					   priv->children[child_index],
-					   priv->children[child_index]->fops->entrylk,
-					   this->name,
-                                           &local->transaction.parent_loc,
-					   local->transaction.basename,
-					   ENTRYLK_LOCK, ENTRYLK_WRLCK);
-		}
-
-		break;
-	}
-
-	return 0;
-
-
+        case AFR_ENTRY_RENAME_TRANSACTION:
+            if (unlock) {
+                cmd = ENTRYLK_UNLOCK;
+            } else if (blocking) { /*Doesn't make sense to have blocking
+                                      unlock*/
+                cmd = ENTRYLK_LOCK;
+            }
+
+            if (local->fd) {
+                STACK_WIND_COOKIE(frame, cbk, cookie, priv->children[child],
+                                  priv->children[child]->fops->fentrylk,
+                                  int_lock->domain,
+                                  int_lock->lockee[lockee_num].fd,
+                                  int_lock->lockee[lockee_num].basename, cmd,
+                                  ENTRYLK_WRLCK, NULL);
+            } else {
+                STACK_WIND_COOKIE(frame, cbk, cookie, priv->children[child],
+                                  priv->children[child]->fops->entrylk,
+                                  int_lock->domain,
+                                  &int_lock->lockee[lockee_num].loc,
+                                  int_lock->lockee[lockee_num].basename, cmd,
+                                  ENTRYLK_WRLCK, NULL);
+            }
+            break;
+
+        case AFR_DATA_TRANSACTION:
+        case AFR_METADATA_TRANSACTION:
+            flock = int_lock->lockee[lockee_num].flock;
+            if (unlock) {
+                flock.l_type = F_UNLCK;
+            } else if (blocking) { /*Doesn't make sense to have blocking
+                                      unlock*/
+                cmd1 = F_SETLKW;
+            }
+
+            if (local->fd) {
+                STACK_WIND_COOKIE(
+                    frame, cbk, cookie, priv->children[child],
+                    priv->children[child]->fops->finodelk, int_lock->domain,
+                    int_lock->lockee[lockee_num].fd, cmd1, &flock, NULL);
+            } else {
+                STACK_WIND_COOKIE(
+                    frame, cbk, cookie, priv->children[child],
+                    priv->children[child]->fops->inodelk, int_lock->domain,
+                    &int_lock->lockee[lockee_num].loc, cmd1, &flock, NULL);
+            }
+            break;
+    }
 }
 
-int32_t
-afr_blocking_lock (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_private_t       *priv     = NULL;
-
-        priv     = this->private;
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        switch (local->transaction.type) {
-	case AFR_DATA_TRANSACTION:
-	case AFR_METADATA_TRANSACTION:
-                initialize_inodelk_variables (frame, this);
-                break;
-
-        case AFR_ENTRY_RENAME_TRANSACTION:
-        case AFR_ENTRY_TRANSACTION:
-                initialize_entrylk_variables (frame, this);
+static int
+afr_unlock_now(call_frame_t *frame, xlator_t *this)
+{
+    afr_internal_lock_t *int_lock = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int call_count = 0;
+    int child_index = 0;
+    int lockee_num = 0;
+    int i = -1;
+
+    local = frame->local;
+    int_lock = &local->internal_lock;
+    priv = this->private;
+
+    call_count = afr_lockee_locked_nodes_count(int_lock);
+
+    int_lock->lk_call_count = call_count;
+
+    if (!call_count) {
+        gf_msg_trace(this->name, 0, "No internal locks unlocked");
+        int_lock->lock_cbk(frame, this);
+        goto out;
+    }
+
+    for (i = 0; i < int_lock->lockee_count * priv->child_count; i++) {
+        lockee_num = i / priv->child_count;
+        child_index = i % priv->child_count;
+        if (int_lock->lockee[lockee_num].locked_nodes[child_index] &
+            LOCKED_YES) {
+            afr_internal_lock_wind(frame, afr_unlock_common_cbk,
+                                   (void *)(long)i, child_index, lockee_num,
+                                   _gf_false, _gf_true);
+            if (!--call_count)
                 break;
         }
+    }
 
-        afr_lock_blocking (frame, this, 0);
-
-        return 0;
+out:
+    return 0;
 }
 
 static int32_t
-afr_nonblocking_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                             int32_t op_ret, int32_t op_errno)
-{
-        afr_internal_lock_t *int_lock = NULL;
-	afr_local_t         *local    = NULL;
-	afr_private_t       *priv     = NULL;
-
-        int call_count          = 0;
-	int child_index         = (long) cookie;
-
-	local    = frame->local;
-        int_lock = &local->internal_lock;
-	priv     = this->private;
-
-        afr_trace_entrylk_out (frame, AFR_ENTRYLK_TRANSACTION,
-                               AFR_LOCK_OP, NULL, op_ret,
-                               op_errno, (long) cookie);
-
-        LOCK (&frame->lock);
-        {
-                call_count = --int_lock->lk_call_count;
-        }
-        UNLOCK (&frame->lock);
-
-        if (op_ret < 0 ) {
-			if (op_errno == ENOSYS) {
-				/* return ENOTSUP */
-				gf_log (this->name, GF_LOG_ERROR,
-					"subvolume does not support locking. "
-					"please load features/posix-locks xlator on server");
-				local->op_ret         = op_ret;
-                                int_lock->lock_op_ret = op_ret;
-
-                                local->child_up[child_index] = 0;
-                                int_lock->lock_op_errno      = op_errno;
-                                local->op_errno              = op_errno;
-                        }
-        } else if (op_ret == 0) {
-                        int_lock->entry_locked_nodes[child_index]
-                                |= LOCKED_YES;
-                        int_lock->entrylk_lock_count++;
-        }
-
-        if (call_count == 0) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "Last locking reply received");
-                /* all locks successfull. Proceed to call FOP */
-                if (int_lock->entrylk_lock_count ==
-                    afr_up_children_count (priv->child_count, local->child_up)) {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "All servers locked. Calling the cbk");
-                        int_lock->lock_op_ret = 0;
-                        int_lock->lock_cbk (frame, this);
-                }
-                /* Not all locks were successfull. Unlock and try locking
-                   again, this time with serially blocking locks */
-                else {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "%d servers locked. Trying again with blocking calls",
-                                int_lock->lock_count);
-
-                        afr_unlock(frame, this);
+afr_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, dict_t *xdata)
+{
+    afr_internal_lock_t *int_lock = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int cky = (long)cookie;
+    int child_index = 0;
+    int lockee_num = 0;
+
+    priv = this->private;
+    local = frame->local;
+    int_lock = &local->internal_lock;
+
+    child_index = ((int)cky) % priv->child_count;
+    lockee_num = ((int)cky) / priv->child_count;
+
+    LOCK(&frame->lock);
+    {
+        if (op_ret == -1) {
+            if (op_errno == ENOSYS) {
+                /* return ENOTSUP */
+                gf_msg(this->name, GF_LOG_ERROR, ENOSYS,
+                       AFR_MSG_LOCK_XLATOR_NOT_LOADED,
+                       "subvolume does not support locking. "
+                       "please load features/locks xlator on server");
+                local->op_ret = op_ret;
+                int_lock->lock_op_ret = op_ret;
+            }
+
+            local->op_errno = op_errno;
+            int_lock->lock_op_errno = op_errno;
+        }
+
+        int_lock->lk_attempted_count++;
+    }
+    UNLOCK(&frame->lock);
+
+    if ((op_ret == -1) && (op_errno == ENOSYS)) {
+        afr_unlock_now(frame, this);
+    } else {
+        if (op_ret == 0) {
+            int_lock->lockee[lockee_num]
+                .locked_nodes[child_index] |= LOCKED_YES;
+            int_lock->lockee[lockee_num].locked_count++;
+            int_lock->lock_count++;
+            if (local->transaction.type == AFR_DATA_TRANSACTION) {
+                LOCK(&local->inode->lock);
+                {
+                    local->inode_ctx->lock_count++;
                 }
+                UNLOCK(&local->inode->lock);
+            }
         }
+        afr_lock_blocking(frame, this, cky + 1);
+    }
 
-        return 0;
+    return 0;
 }
 
-int
-afr_nonblocking_entrylk (call_frame_t *frame, xlator_t *this)
+static gf_boolean_t
+_is_lock_wind_needed(afr_local_t *local, int child_index)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_private_t       *priv     = NULL;
-        afr_fd_ctx_t        *fd_ctx   = NULL;
-        const char          *basename = NULL;
-        loc_t               *loc      = NULL;
-
-        int32_t call_count = 0;
-        int i = 0;
-        uint64_t  ctx;
-        int ret = 0;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        priv     = this->private;
-
-        initialize_entrylk_variables (frame, this);
-
-        basename = int_lock->lk_basename;
-        if (int_lock->lk_loc)
-                loc = int_lock->lk_loc;
-
-        if (local->fd) {
-                ret = fd_ctx_get (local->fd, this, &ctx);
-
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "unable to get fd ctx for fd=%p",
-                                local->fd);
-
-                        local->op_ret           = -1;
-                        int_lock->lock_op_ret   = -1;
-                        local->op_errno         = EINVAL;
-                        int_lock->lock_op_errno = EINVAL;
-
-                        return -1;
-                }
-
-                fd_ctx = (afr_fd_ctx_t *)(long) ctx;
-
-                call_count = internal_lock_count (frame, this, fd_ctx);
-                int_lock->lk_call_count = call_count;
+    if (!local->child_up[child_index])
+        return _gf_false;
 
-                if (!call_count) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "fd not open on any subvolumes. aborting.");
-                        afr_unlock (frame, this);
-                        goto out;
-                }
-
-                /* Send non-blocking entrylk calls only on up children
-                   and where the fd has been opened */
-                for (i = 0; i < priv->child_count; i++) {
-                        if (local->child_up[i] && fd_ctx->opened_on[i]) {
-                                afr_trace_entrylk_in (frame, AFR_ENTRYLK_NB_TRANSACTION,
-                                                      AFR_LOCK_OP, basename, i);
-
-                                STACK_WIND_COOKIE (frame, afr_nonblocking_entrylk_cbk,
-                                                   (void *) (long) i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->fentrylk,
-                                                   this->name, local->fd,
-                                                   basename,
-                                                   ENTRYLK_LOCK_NB, ENTRYLK_WRLCK);
-                        }
-                }
-        } else {
-                GF_ASSERT (loc);
-
-                call_count = internal_lock_count (frame, this, NULL);
-                int_lock->lk_call_count = call_count;
-
-                for (i = 0; i < priv->child_count; i++) {
-                        if (local->child_up[i]) {
-                                afr_trace_entrylk_in (frame, AFR_ENTRYLK_NB_TRANSACTION,
-                                                      AFR_LOCK_OP, basename, i);
-
-                                STACK_WIND_COOKIE (frame, afr_nonblocking_entrylk_cbk,
-                                                   (void *) (long) i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->entrylk,
-                                                   this->name, loc, basename,
-                                                   ENTRYLK_LOCK, ENTRYLK_WRLCK);
-
-                                if (!--call_count)
-                                        break;
-
-                        }
-                }
-        }
-out:
-        return 0;
+    return _gf_true;
 }
 
-int32_t
-afr_nonblocking_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                             int32_t op_ret, int32_t op_errno)
+static gf_boolean_t
+is_blocking_locks_count_sufficient(call_frame_t *frame, xlator_t *this)
 {
-        afr_internal_lock_t *int_lock = NULL;
-	afr_local_t         *local    = NULL;
-	afr_private_t       *priv     = NULL;
-
-        int call_count  = 0;
-	int child_index = (long) cookie;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    afr_internal_lock_t *int_lock = NULL;
+    int child = 0;
+    int nlockee = 0;
+    int lockee_count = 0;
+    gf_boolean_t ret = _gf_true;
 
-	local    = frame->local;
-        int_lock = &local->internal_lock;
-	priv     = this->private;
-
-        afr_trace_inodelk_out (frame, AFR_INODELK_NB_TRANSACTION,
-                               AFR_LOCK_OP, NULL, op_ret,
-                               op_errno, (long) cookie);
-
-        LOCK (&frame->lock);
-        {
-                call_count = --int_lock->lk_call_count;
-        }
-        UNLOCK (&frame->lock);
-
-        if (op_ret < 0 ) {
-			if (op_errno == ENOSYS) {
-				/* return ENOTSUP */
-				gf_log (this->name, GF_LOG_ERROR,
-					"subvolume does not support locking. "
-					"please load features/posix-locks xlator on server");
-				local->op_ret                = op_ret;
-                                int_lock->lock_op_ret        = op_ret;
-                                local->child_up[child_index] = 0;
-                                int_lock->lock_op_errno      = op_errno;
-                                local->op_errno              = op_errno;
-                        }
-        } else if (op_ret == 0) {
-                        int_lock->inode_locked_nodes[child_index]
-                                |= LOCKED_YES;
-                        int_lock->inodelk_lock_count++;
-        }
+    local = frame->local;
+    priv = this->private;
+    int_lock = &local->internal_lock;
+    lockee_count = int_lock->lockee_count;
 
-        if (call_count == 0) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "Last inode locking reply received");
-                /* all locks successfull. Proceed to call FOP */
-                if (int_lock->inodelk_lock_count ==
-                    afr_up_children_count (priv->child_count, local->child_up)) {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "All servers locked. Calling the cbk");
-                        int_lock->lock_op_ret = 0;
-                        int_lock->lock_cbk (frame, this);
-                }
-                /* Not all locks were successfull. Unlock and try locking
-                   again, this time with serially blocking locks */
-                else {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "%d servers locked. Trying again with blocking calls",
-                                int_lock->lock_count);
-
-                        afr_unlock(frame, this);
-                }
+    if (int_lock->lock_count == 0) {
+        afr_log_locks_failure(frame, "any subvolume", "lock",
+                              int_lock->lock_op_errno);
+        return _gf_false;
+    }
+    /* For FOPS that take multiple sets of locks (mkdir, rename),
+     * there must be at least one brick on which the locks from
+     * all lock sets were successful. */
+    for (child = 0; child < priv->child_count; child++) {
+        ret = _gf_true;
+        for (nlockee = 0; nlockee < lockee_count; nlockee++) {
+            if (!(int_lock->lockee[nlockee].locked_nodes[child] & LOCKED_YES))
+                ret = _gf_false;
         }
+        if (ret)
+            return ret;
+    }
+    if (!ret)
+        afr_log_locks_failure(frame, "all", "lock", int_lock->lock_op_errno);
 
-        return 0;
+    return ret;
 }
 
 int
-afr_nonblocking_inodelk (call_frame_t *frame, xlator_t *this)
+afr_lock_blocking(call_frame_t *frame, xlator_t *this, int cookie)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_private_t       *priv     = NULL;
-        afr_fd_ctx_t        *fd_ctx   = NULL;
-
-        int32_t  call_count = 0;
-        uint64_t ctx        = 0;
-        int      i          = 0;
-        int      ret        = 0;
-        struct gf_flock flock;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        priv     = this->private;
-
-	flock.l_start = int_lock->lk_flock.l_start;
-	flock.l_len   = int_lock->lk_flock.l_len;
-	flock.l_type  = int_lock->lk_flock.l_type;
-
-        initialize_inodelk_variables (frame, this);
-
-        if (local->fd) {
-                ret = fd_ctx_get (local->fd, this, &ctx);
-
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "unable to get fd ctx for fd=%p",
-                                local->fd);
-
-                        local->op_ret           = -1;
-                        int_lock->lock_op_ret   = -1;
-                        local->op_errno         = EINVAL;
-                        int_lock->lock_op_errno = EINVAL;
-
-                        ret = -1;
-                        goto out;
-                }
+    afr_internal_lock_t *int_lock = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    uint64_t ctx = 0;
+    int ret = 0;
+    int child_index = 0;
+    int lockee_num = 0;
 
-                fd_ctx = (afr_fd_ctx_t *)(long) ctx;
+    local = frame->local;
+    int_lock = &local->internal_lock;
+    priv = this->private;
+    child_index = cookie % priv->child_count;
+    lockee_num = cookie / priv->child_count;
 
-                call_count = internal_lock_count (frame, this, fd_ctx);
-                int_lock->lk_call_count = call_count;
-
-                if (!call_count) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "fd not open on any subvolumes. aborting.");
-                        afr_unlock (frame, this);
-                        goto out;
-                }
+    if (local->fd) {
+        ret = fd_ctx_get(local->fd, this, &ctx);
 
-                /* Send non-blocking inodelk calls only on up children
-                   and where the fd has been opened */
-                for (i = 0; i < priv->child_count; i++) {
-                        if (local->child_up[i] && fd_ctx->opened_on[i]) {
-                                afr_trace_inodelk_in (frame, AFR_INODELK_NB_TRANSACTION,
-                                                      AFR_LOCK_OP, &flock, F_SETLK, i);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_FD_CTX_GET_FAILED,
+                   "unable to get fd ctx for fd=%p", local->fd);
 
-                                STACK_WIND_COOKIE (frame, afr_nonblocking_inodelk_cbk,
-                                                   (void *) (long) i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->finodelk,
-                                                   this->name, local->fd,
-                                                   F_SETLK, &flock);
+            local->op_ret = -1;
+            int_lock->lock_op_ret = -1;
 
-                                if (!--call_count)
-                                        break;
+            afr_unlock_now(frame, this);
 
-                        }
-
-                }
-        } else {
-                call_count = internal_lock_count (frame, this, NULL);
-                int_lock->lk_call_count = call_count;
-
-                for (i = 0; i < priv->child_count; i++) {
-                        if (local->child_up[i]) {
-                                afr_trace_inodelk_in (frame, AFR_INODELK_NB_TRANSACTION,
-                                                      AFR_LOCK_OP, &flock, F_SETLK, i);
-
-                                STACK_WIND_COOKIE (frame, afr_nonblocking_inodelk_cbk,
-                                                   (void *) (long) i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->inodelk,
-                                                   this->name, &local->loc,
-                                                   F_SETLK, &flock);
-
-                                if (!--call_count)
-                                        break;
-
-                        }
-                }
+            return 0;
         }
+    }
 
-out:
-        return ret;
-}
-
-static int
-__is_lower_locked (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_private_t       *priv     = NULL;
-        afr_local_t         *local    = NULL;
-
-        int count = 0;
-        int i     = 0;
+    if (int_lock->lk_expected_count == int_lock->lk_attempted_count) {
+        if (!is_blocking_locks_count_sufficient(frame, this)) {
+            local->op_ret = -1;
+            int_lock->lock_op_ret = -1;
 
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        priv     = this->private;
+            afr_unlock_now(frame, this);
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (int_lock->lower_locked_nodes[i] & LOCKED_LOWER)
-                        count++;
+            return 0;
         }
+    }
 
-        return count;
+    if (int_lock->lk_expected_count == int_lock->lk_attempted_count) {
+        /* we're done locking */
 
-}
-
-static int
-__is_higher_locked (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_private_t       *priv     = NULL;
-        afr_local_t         *local    = NULL;
-
-        int count = 0;
-        int i = 0;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        priv     = this->private;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (int_lock->locked_nodes[i] & LOCKED_YES)
-                        count++;
-        }
+        gf_msg_debug(this->name, 0, "we're done locking");
 
-        return count;
-
-}
-
-static int
-afr_unlock_lower_entrylk (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_private_t       *priv     = NULL;
-        const char          *basename = NULL;
-        loc_t               *loc      = NULL;
-
-        int call_count = 0;
-        int i = -1;
-
-        local = frame->local;
-        int_lock = &local->internal_lock;
-        priv  = this->private;
-
-        basename = int_lock->lk_basename;
-        if (int_lock->lk_loc)
-                loc = int_lock->lk_loc;
-
-        call_count = __is_lower_locked (frame, this);
-        int_lock->lk_call_count = call_count;
-
-        if (!call_count){
-                gf_log (this->name, GF_LOG_TRACE,
-                        "No internal locks unlocked");
-                int_lock->lock_cbk (frame, this);
-                goto out;
-        }
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (int_lock->lower_locked_nodes[i] & LOCKED_LOWER) {
-                        afr_trace_entrylk_in (frame, AFR_ENTRYLK_NB_TRANSACTION,
-                                              AFR_UNLOCK_OP, basename, i);
-
-                        STACK_WIND_COOKIE (frame, afr_unlock_entrylk_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->entrylk,
-                                           this->name,
-                                           loc, basename,
-                                           ENTRYLK_UNLOCK, ENTRYLK_WRLCK);
-
-                        if (!--call_count)
-                                break;
-
-                }
-        }
-
-out:
+        int_lock->lock_op_ret = 0;
+        int_lock->lock_cbk(frame, this);
         return 0;
+    }
 
-}
-
-
-static int
-afr_post_unlock_higher_cbk (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local = NULL;
-
-        local = frame->local;
-
-        local->transaction.done (frame, this);
-        return 0;
-}
-
-static int
-afr_post_unlock_lower_cbk (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock    = NULL;
-        afr_local_t         *local       = NULL;
-	loc_t               *lower       = NULL;
-	loc_t               *higher      = NULL;
-	const char          *lower_name  = NULL;
-	const char          *higher_name = NULL;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        lower = lower_path (&local->transaction.parent_loc,
-                            local->transaction.basename,
-                            &local->transaction.new_parent_loc,
-                            local->transaction.new_basename);
-
-        lower_name = (lower == &local->transaction.parent_loc ?
-                      local->transaction.basename :
-                      local->transaction.new_basename);
-
-        higher = (lower == &local->transaction.parent_loc ?
-                  &local->transaction.new_parent_loc :
-                  &local->transaction.parent_loc);
-
-        higher_name = (higher == &local->transaction.parent_loc ?
-                       local->transaction.basename :
-                       local->transaction.new_basename);
-
-        if (__is_higher_locked (frame, this)) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "unlocking higher");
-                int_lock->lk_basename = higher_name;
-                int_lock->lk_loc      = higher;
-                int_lock->lock_cbk    = afr_post_unlock_higher_cbk;
-
-                afr_unlock_entrylk (frame, this);
-        } else
-                local->transaction.done (frame, this);
-
-        return 0;
-}
-
-static int
-afr_rename_unlock (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock    = NULL;
-        afr_local_t         *local       = NULL;
-	loc_t               *lower       = NULL;
-	loc_t               *higher      = NULL;
-	const char          *lower_name  = NULL;
-	const char          *higher_name = NULL;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        lower = lower_path (&local->transaction.parent_loc,
-                            local->transaction.basename,
-                            &local->transaction.new_parent_loc,
-                            local->transaction.new_basename);
-
-        lower_name = (lower == &local->transaction.parent_loc ?
-                      local->transaction.basename :
-                      local->transaction.new_basename);
-
-        higher = (lower == &local->transaction.parent_loc ?
-                  &local->transaction.new_parent_loc :
-                  &local->transaction.parent_loc);
-
-        higher_name = (higher == &local->transaction.parent_loc ?
-                       local->transaction.basename :
-                       local->transaction.new_basename);
-
-
-        if (__is_lower_locked (frame, this)) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "unlocking lower");
-                int_lock->lk_basename = lower_name;
-                int_lock->lk_loc      = lower;
-                int_lock->lock_cbk    = afr_post_unlock_lower_cbk;
-
-                afr_unlock_lower_entrylk (frame, this);
-        } else
-                afr_post_unlock_lower_cbk (frame, this);
-
+    if (!_is_lock_wind_needed(local, child_index)) {
+        afr_lock_blocking(frame, this, cookie + 1);
         return 0;
-}
-
-static int
-afr_rename_transaction (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local = NULL;
-
-        local = frame->local;
+    }
 
-        return (local->transaction.type ==
-                AFR_ENTRY_RENAME_TRANSACTION);
+    afr_internal_lock_wind(frame, afr_lock_cbk, (void *)(long)cookie,
+                           child_index, lockee_num, _gf_true, _gf_false);
 
+    return 0;
 }
 
 int32_t
-afr_unlock (call_frame_t *frame, xlator_t *this)
+afr_blocking_lock(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *local = NULL;
-
-        local = frame->local;
-
-        if (transaction_lk_op (local)) {
-                if (is_afr_lock_transaction (local))
-                        afr_unlock_inodelk (frame, this);
-                else
-                        if (!afr_rename_transaction (frame, this))
-                                afr_unlock_entrylk (frame, this);
-                        else
-                                afr_rename_unlock (frame, this);
-        } else {
-                if (is_afr_lock_selfheal (local))
-                        afr_unlock_inodelk (frame, this);
-                else
-                        afr_unlock_entrylk (frame, this);
-        }
+    afr_internal_lock_t *int_lock = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int up_count = 0;
 
-        return 0;
-}
+    priv = this->private;
+    local = frame->local;
+    int_lock = &local->internal_lock;
 
-int
-afr_mark_locked_nodes (xlator_t *this, fd_t *fd,
-                       unsigned char *locked_nodes)
-{
-        afr_private_t *priv  = NULL;
-        afr_fd_ctx_t  *fdctx = NULL;
-        uint64_t       tmp   = 0;
-        int            ret   = 0;
+    up_count = AFR_COUNT(local->child_up, priv->child_count);
+    int_lock->lk_call_count = int_lock->lk_expected_count =
+        (int_lock->lockee_count * up_count);
+    initialize_internal_lock_variables(frame, this);
 
-        priv = this->private;
-
-        ret = afr_fd_ctx_set (this, fd);
-        if (ret)
-                goto out;
+    afr_lock_blocking(frame, this, 0);
 
-        ret = fd_ctx_get (fd, this, &tmp);
-        fdctx = (afr_fd_ctx_t *) (long) tmp;
-
-        GF_ASSERT (fdctx->locked_on);
-
-        memcpy (fdctx->locked_on, locked_nodes,
-                priv->child_count);
-
-out:
-        return ret;
+    return 0;
 }
 
-static int
-__is_fd_saved (xlator_t *this, fd_t *fd)
+static int32_t
+afr_nb_internal_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        afr_locked_fd_t *locked_fd = NULL;
-        afr_private_t   *priv      = NULL;
-        int              found     = 0;
+    afr_internal_lock_t *int_lock = NULL;
+    afr_local_t *local = NULL;
+    int call_count = 0;
+    int child_index = 0;
+    int lockee_num = 0;
+    afr_private_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        list_for_each_entry (locked_fd, &priv->saved_fds, list) {
-                if (locked_fd->fd == fd) {
-                        found = 1;
-                        break;
-                }
-        }
+    child_index = ((long)cookie) % priv->child_count;
+    lockee_num = ((long)cookie) / priv->child_count;
 
-        return found;
-}
+    local = frame->local;
+    int_lock = &local->internal_lock;
 
-static int
-__afr_save_locked_fd (xlator_t *this, fd_t *fd)
-{
-        afr_private_t   *priv      = NULL;
-        afr_locked_fd_t *locked_fd = NULL;
-        int              ret       = 0;
-
-        priv = this->private;
-
-        locked_fd = GF_CALLOC (1, sizeof (*locked_fd),
-                               gf_afr_mt_locked_fd);
-        if (!locked_fd) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory");
-                ret = -1;
-                goto out;
-        }
-
-        locked_fd->fd = fd;
-        INIT_LIST_HEAD (&locked_fd->list);
-
-        list_add_tail (&locked_fd->list, &priv->saved_fds);
-
-out:
-        return ret;
-}
-
-int
-afr_save_locked_fd (xlator_t *this, fd_t *fd)
-{
-        afr_private_t   *priv      = NULL;
-        int              ret       = 0;
-
-        priv = this->private;
-
-        pthread_mutex_lock (&priv->mutex);
+    if (op_ret == 0 && local->transaction.type == AFR_DATA_TRANSACTION) {
+        LOCK(&local->inode->lock);
         {
-                if (__is_fd_saved (this, fd)) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "fd=%p already saved", fd);
-                        goto unlock;
-                }
-
-                ret = __afr_save_locked_fd (this, fd);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "fd=%p could not be saved", fd);
-                        goto unlock;
-                }
+            local->inode_ctx->lock_count++;
         }
-unlock:
-        pthread_mutex_unlock (&priv->mutex);
-
-        return ret;
-}
-
-static int
-afr_lock_recovery_cleanup (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t     *local     = NULL;
-        afr_locked_fd_t *locked_fd = NULL;
-
-        local = frame->local;
-
-        locked_fd = local->locked_fd;
-
-        STACK_DESTROY (frame->root);
-        afr_local_cleanup (local, this);
-
-        afr_save_locked_fd (this, locked_fd->fd);
-
-        return 0;
-
-}
-
-static int
-afr_get_source_lock_recovery (xlator_t *this, fd_t *fd)
-{
-        afr_fd_ctx_t  *fdctx        = NULL;
-        afr_private_t *priv         = NULL;
-        uint64_t      tmp           = 0;
-        int           i             = 0;
-        int           source_child  = -1;
-        int           ret           = 0;
-
-        priv = this->private;
-
-        ret = fd_ctx_get (fd, this, &tmp);
-        if (ret)
-                goto out;
-
-        fdctx = (afr_fd_ctx_t *) (long) tmp;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (fdctx->locked_on[i]) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "Found lock recovery source=%d",
-                                i);
-                        source_child = i;
-                        break;
-                }
-
-        }
-
-out:
-        return source_child;
-
-}
+        UNLOCK(&local->inode->lock);
+    }
 
-int32_t
-afr_get_locks_fd_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct gf_flock *lock);
-int32_t
-afr_recover_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct gf_flock *lock)
-{
-        afr_local_t   *local = NULL;
-        afr_private_t *priv  = NULL;
-        int32_t source_child = 0;
-        struct gf_flock flock   = {0,};
-
-        local = frame->local;
-        priv  = this->private;
-
-        if (op_ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "lock recovery failed");
-                goto cleanup;
+    LOCK(&frame->lock);
+    {
+        if (op_ret < 0) {
+            if (op_errno == ENOSYS) {
+                /* return ENOTSUP */
+                gf_msg(this->name, GF_LOG_ERROR, ENOSYS,
+                       AFR_MSG_LOCK_XLATOR_NOT_LOADED,
+                       "subvolume does not support "
+                       "locking. please load features/locks"
+                       " xlator on server");
+                local->op_ret = op_ret;
+                int_lock->lock_op_ret = op_ret;
+
+                int_lock->lock_op_errno = op_errno;
+                local->op_errno = op_errno;
+            }
+        } else if (op_ret == 0) {
+            int_lock->lockee[lockee_num]
+                .locked_nodes[child_index] |= LOCKED_YES;
+            int_lock->lockee[lockee_num].locked_count++;
+            int_lock->lock_count++;
         }
 
-        source_child = local->source_child;
-
-        memcpy (&flock, lock, sizeof (*lock));
-
-        STACK_WIND_COOKIE (frame, afr_get_locks_fd_cbk,
-                           (void *) (long) source_child,
-                           priv->children[source_child],
-                           priv->children[source_child]->fops->lk,
-                           local->fd, F_GETLK_FD, &flock);
-
-        return 0;
-
-cleanup:
-        afr_lock_recovery_cleanup (frame, this);
-        return 0;
-}
-
-int
-afr_recover_lock (call_frame_t *frame, xlator_t *this,
-                  struct gf_flock *flock)
-{
-        afr_local_t   *local             = NULL;
-        afr_private_t *priv              = NULL;
-        int32_t      lock_recovery_child = 0;
-
-        priv  = this->private;
-        local = frame->local;
-
-        lock_recovery_child = local->lock_recovery_child;
+        call_count = --int_lock->lk_call_count;
+    }
+    UNLOCK(&frame->lock);
 
-        frame->root->lk_owner = flock->l_owner;
-
-        STACK_WIND_COOKIE (frame, afr_recover_lock_cbk,
-                           (void *) (long) lock_recovery_child,
-                           priv->children[lock_recovery_child],
-                           priv->children[lock_recovery_child]->fops->lk,
-                           local->fd, F_SETLK, flock);
-
-        return 0;
-}
-
-static int
-is_afr_lock_eol (struct gf_flock *lock)
-{
-        int ret = 0;
-
-        if ((lock->l_type == GF_LK_EOL))
-                ret = 1;
-
-        return ret;
-}
-
-int32_t
-afr_get_locks_fd_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct gf_flock *lock)
-{
-        if (op_ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Failed to get locks on fd");
-                goto cleanup;
+    if (call_count == 0) {
+        gf_msg_trace(this->name, 0, "Last locking reply received");
+        /* all locks successful. Proceed to call FOP */
+        if (int_lock->lock_count == int_lock->lk_expected_count) {
+            gf_msg_trace(this->name, 0, "All servers locked. Calling the cbk");
+            int_lock->lock_op_ret = 0;
+            int_lock->lock_cbk(frame, this);
         }
+        /* Not all locks were successful. Unlock and try locking
+           again, this time with serially blocking locks */
+        else {
+            gf_msg_trace(this->name, 0,
+                         "%d servers locked. Trying again "
+                         "with blocking calls",
+                         int_lock->lock_count);
 
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Got a lock on fd");
-
-        if (is_afr_lock_eol (lock)) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Reached EOL on locks on fd");
-                goto cleanup;
+            afr_unlock_now(frame, this);
         }
+    }
 
-        afr_recover_lock (frame, this, lock);
-
-        return 0;
-
-cleanup:
-        afr_lock_recovery_cleanup (frame, this);
-
-        return 0;
+    return 0;
 }
 
-static int
-afr_lock_recovery (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t   *local        = NULL;
-        afr_private_t *priv         = NULL;
-        fd_t          *fd           = NULL;
-        int            ret          = 0;
-        int32_t        source_child = 0;
-        struct gf_flock   flock        = {0,};
-
-        priv  = this->private;
-        local = frame->local;
-
-        fd = local->fd;
-
-        source_child = afr_get_source_lock_recovery (this, fd);
-        if (source_child < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Could not recover locks due to lock "
-                        "split brain");
-                ret = -1;
-                goto out;
+int
+afr_lock_nonblocking(call_frame_t *frame, xlator_t *this)
+{
+    afr_internal_lock_t *int_lock = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    afr_fd_ctx_t *fd_ctx = NULL;
+    int child = 0;
+    int lockee_num = 0;
+    int32_t call_count = 0;
+    int i = 0;
+    int ret = 0;
+
+    local = frame->local;
+    int_lock = &local->internal_lock;
+    priv = this->private;
+
+    initialize_internal_lock_variables(frame, this);
+
+    if (local->fd) {
+        fd_ctx = afr_fd_ctx_get(local->fd, this);
+        if (!fd_ctx) {
+            gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_FD_CTX_GET_FAILED,
+                   "unable to get fd ctx for fd=%p", local->fd);
+
+            local->op_ret = -1;
+            int_lock->lock_op_ret = -1;
+            local->op_errno = EINVAL;
+            int_lock->lock_op_errno = EINVAL;
+
+            afr_unlock_now(frame, this);
+            ret = -1;
+            goto out;
+        }
+    }
+
+    call_count = int_lock->lockee_count * internal_lock_count(frame, this);
+    int_lock->lk_call_count = call_count;
+    int_lock->lk_expected_count = call_count;
+
+    if (!call_count) {
+        gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_INFO_COMMON,
+               "fd not open on any subvolumes. aborting.");
+        afr_unlock_now(frame, this);
+        goto out;
+    }
+
+    /* Send non-blocking lock calls only on up children
+       and where the fd has been opened */
+    for (i = 0; i < int_lock->lockee_count * priv->child_count; i++) {
+        child = i % priv->child_count;
+        lockee_num = i / priv->child_count;
+        if (local->child_up[child]) {
+            afr_internal_lock_wind(frame, afr_nb_internal_lock_cbk,
+                                   (void *)(long)i, child, lockee_num,
+                                   _gf_false, _gf_false);
+            if (!--call_count)
+                break;
         }
-
-        local->source_child = source_child;
-
-        /* the flock can be zero filled as we're querying incrementally
-           the locks held on the fd.
-        */
-        STACK_WIND_COOKIE (frame, afr_get_locks_fd_cbk,
-                           (void *) (long) source_child,
-                           priv->children[source_child],
-                           priv->children[source_child]->fops->lk,
-                           local->fd, F_GETLK_FD, &flock);
-
+    }
 out:
-        return ret;
-}
-
-
-static int
-afr_mark_fd_opened (xlator_t *this, fd_t *fd, int32_t child_index)
-{
-        afr_fd_ctx_t *fdctx = NULL;
-        uint64_t      tmp   = 0;
-        int           ret   = 0;
-
-        ret = fd_ctx_get (fd, this, &tmp);
-        if (ret)
-                goto out;
-
-        fdctx = (afr_fd_ctx_t *) (long) tmp;
-
-        fdctx->opened_on[child_index] = 1;
-
-out:
-        return ret;
+    return ret;
 }
 
 int32_t
-afr_lock_recovery_preopen_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                               int32_t op_ret, int32_t op_errno, fd_t *fd)
-{
-        int32_t child_index = (long )cookie;
-        int ret = 0;
-
-        if (op_ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Reopen during lock-recovery failed");
-                goto cleanup;
-        }
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Open succeeded => proceed to recover locks");
-
-        ret = afr_lock_recovery (frame, this);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Lock recovery failed");
-                goto cleanup;
-        }
-
-        ret = afr_mark_fd_opened (this, fd, child_index);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Marking fd open failed");
-                goto cleanup;
-        }
-
-        return 0;
-
-cleanup:
-        afr_lock_recovery_cleanup (frame, this);
-        return 0;
-}
-
-static int
-afr_lock_recovery_preopen (call_frame_t *frame, xlator_t *this)
-{
-        afr_private_t *priv  = NULL;
-        afr_local_t   *local = NULL;
-        uint64_t       tmp   = 0;
-        afr_fd_ctx_t  *fdctx = NULL;
-        loc_t          loc   = {0,};
-        int32_t        child_index = 0;
-        int            ret   = 0;
-
-        priv  = this->private;
-        local = frame->local;
-
-        GF_ASSERT (local && local->fd);
-
-        ret = fd_ctx_get (local->fd, this, &tmp);
-        fdctx = (afr_fd_ctx_t *) (long) tmp;
-        GF_ASSERT (fdctx);
-
-        child_index = local->lock_recovery_child;
-
-        inode_path (local->fd->inode, NULL, (char **)&loc.path);
-        loc.name   = strrchr (loc.path, '/');
-        loc.inode  = inode_ref (local->fd->inode);
-        loc.parent = inode_parent (local->fd->inode, 0, NULL);
-
-
-        STACK_WIND_COOKIE (frame, afr_lock_recovery_preopen_cbk,
-                           (void *)(long) child_index,
-                           priv->children[child_index],
-                           priv->children[child_index]->fops->open,
-                           &loc, fdctx->flags, local->fd,
-                           fdctx->wbflags);
-
+afr_unlock(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_lock_t *lock = NULL;
+
+    local = frame->local;
+
+    if (!local->transaction.eager_lock_on)
+        goto out;
+    lock = &local->inode_ctx->lock[local->transaction.type];
+    LOCK(&local->inode->lock);
+    {
+        list_del_init(&local->transaction.owner_list);
+        if (list_empty(&lock->owners) && list_empty(&lock->post_op)) {
+            local->transaction.do_eager_unlock = _gf_true;
+            /*TODO: Need to get metadata use on_disk and inherit/uninherit
+             *GF_ASSERT (!local->inode_ctx->on_disk[local->transaction.type]);
+             *GF_ASSERT (!local->inode_ctx->inherited[local->transaction.type]);
+             */
+            GF_ASSERT(lock->release);
+        }
+    }
+    UNLOCK(&local->inode->lock);
+    if (!local->transaction.do_eager_unlock) {
+        local->internal_lock.lock_cbk(frame, this);
         return 0;
-}
-
-static int
-is_fd_opened (fd_t *fd, int32_t child_index)
-{
-        afr_fd_ctx_t *fdctx = NULL;
-        uint64_t      tmp = 0;
-        int           ret = 0;
-
-        ret = fd_ctx_get (fd, THIS, &tmp);
-        if (ret)
-                goto out;
-
-        fdctx = (afr_fd_ctx_t *) (long) tmp;
-
-        if (fdctx->opened_on[child_index])
-                ret = 1;
-
-out:
-        return ret;
-}
-
-int
-afr_attempt_lock_recovery (xlator_t *this, int32_t child_index)
-{
-        call_frame_t    *frame     = NULL;
-        afr_private_t   *priv      = NULL;
-        afr_local_t     *local     = NULL;
-        afr_locked_fd_t *locked_fd = NULL;
-       afr_locked_fd_t  *tmp       = NULL;
-        int              ret       = 0;
-        struct list_head locks_list;
-
-
-        priv = this->private;
-
-        if (list_empty (&priv->saved_fds))
-                goto out;
-
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory");
-                ret = -1;
-                goto out;
-        }
-
-        local = GF_CALLOC (1, sizeof (*local),
-                           gf_afr_mt_afr_local_t);
-        if (!local) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Out of memory");
-                ret = -1;
-                goto out;
-        }
-
-        AFR_LOCAL_INIT (local, priv);
-        if (!local) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory");
-                ret = -1;
-                goto out;
-        }
-
-        frame->local = local;
-
-        INIT_LIST_HEAD (&locks_list);
-
-        pthread_mutex_lock (&priv->mutex);
-        {
-                list_splice_init (&priv->saved_fds, &locks_list);
-        }
-        pthread_mutex_unlock (&priv->mutex);
-
-        list_for_each_entry_safe (locked_fd, tmp,
-                                  &locks_list, list) {
-
-                list_del_init (&locked_fd->list);
-
-                local->fd                  = fd_ref (locked_fd->fd);
-                local->lock_recovery_child = child_index;
-                local->locked_fd           = locked_fd;
-
-                if (!is_fd_opened (locked_fd->fd, child_index)) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "attempting open before lock "
-                                "recovery");
-                        afr_lock_recovery_preopen (frame, this);
-                } else {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "attempting lock recovery "
-                                "without a preopen");
-                        afr_lock_recovery (frame, this);
-                }
-        }
+    }
 
 out:
-        return ret;
+    afr_unlock_now(frame, this);
+    return 0;
 }
diff --git a/xlators/cluster/afr/src/afr-mem-types.h b/xlators/cluster/afr/src/afr-mem-types.h
index 14064ebcd76..816065fb57a 100644
--- a/xlators/cluster/afr/src/afr-mem-types.h
+++ b/xlators/cluster/afr/src/afr-mem-types.h
@@ -1,48 +1,38 @@
 /*
-   Copyright (c) 2008-2010 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
-
 #ifndef __AFR_MEM_TYPES_H__
 #define __AFR_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_afr_mem_types_ {
-        gf_afr_mt_iovec  = gf_common_mt_end + 1,
-        gf_afr_mt_afr_fd_ctx_t,
-        gf_afr_mt_afr_local_t,
-        gf_afr_mt_afr_private_t,
-        gf_afr_mt_int32_t,
-        gf_afr_mt_char,
-        gf_afr_mt_xattr_key,
-        gf_afr_mt_dict_t,
-        gf_afr_mt_xlator_t,
-        gf_afr_mt_iatt,
-        gf_afr_mt_int,
-        gf_afr_mt_afr_node_character,
-        gf_afr_mt_sh_diff_loop_state,
-        gf_afr_mt_uint8_t,
-        gf_afr_mt_loc_t,
-        gf_afr_mt_entry_name,
-        gf_afr_mt_pump_priv,
-        gf_afr_mt_locked_fd,
-        gf_afr_mt_end
+    gf_afr_mt_afr_fd_ctx_t = gf_common_mt_end + 1,
+    gf_afr_mt_afr_private_t,
+    gf_afr_mt_int32_t,
+    gf_afr_mt_char,
+    gf_afr_mt_xattr_key,
+    gf_afr_mt_dict_t,
+    gf_afr_mt_xlator_t,
+    gf_afr_mt_afr_node_character,
+    gf_afr_mt_inode_ctx_t,
+    gf_afr_mt_shd_event_t,
+    gf_afr_mt_reply_t,
+    gf_afr_mt_subvol_healer_t,
+    gf_afr_mt_spbc_timeout_t,
+    gf_afr_mt_spb_status_t,
+    gf_afr_mt_empty_brick_t,
+    gf_afr_mt_child_latency_t,
+    gf_afr_mt_atomic_t,
+    gf_afr_mt_lk_heal_info_t,
+    gf_afr_mt_gf_lock,
+    gf_afr_mt_end
 };
 #endif
-
diff --git a/xlators/cluster/afr/src/afr-messages.h b/xlators/cluster/afr/src/afr-messages.h
new file mode 100644
index 00000000000..e73fd997765
--- /dev/null
+++ b/xlators/cluster/afr/src/afr-messages.h
@@ -0,0 +1,167 @@
+/*
+ Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+ This file is part of GlusterFS.
+
+ This file is licensed to you under your choice of the GNU Lesser
+ General Public License, version 3 or any later version (LGPLv3 or
+ later), or the GNU General Public License, version 2 (GPLv2), in all
+ cases as published by the Free Software Foundation.
+ */
+
+#ifndef _AFR_MESSAGES_H_
+#define _AFR_MESSAGES_H_
+
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+GLFS_MSGID(
+    AFR, AFR_MSG_QUORUM_FAIL, AFR_MSG_QUORUM_MET, AFR_MSG_QUORUM_OVERRIDE,
+    AFR_MSG_INVALID_CHILD_UP, AFR_MSG_SUBVOL_UP, AFR_MSG_SUBVOLS_DOWN,
+    AFR_MSG_ENTRY_UNLOCK_FAIL, AFR_MSG_SPLIT_BRAIN, AFR_MSG_OPEN_FAIL,
+    AFR_MSG_UNLOCK_FAIL, AFR_MSG_REPLACE_BRICK_STATUS, AFR_MSG_GFID_NULL,
+    AFR_MSG_FD_CREATE_FAILED, AFR_MSG_DICT_SET_FAILED,
+    AFR_MSG_EXPUNGING_FILE_OR_DIR, AFR_MSG_MIGRATION_IN_PROGRESS,
+    AFR_MSG_CHILD_MISCONFIGURED, AFR_MSG_VOL_MISCONFIGURED,
+    AFR_MSG_INTERNAL_LKS_FAILED, AFR_MSG_INVALID_FD, AFR_MSG_LOCK_INFO,
+    AFR_MSG_LOCK_XLATOR_NOT_LOADED, AFR_MSG_FD_CTX_GET_FAILED,
+    AFR_MSG_INVALID_SUBVOL, AFR_MSG_PUMP_XLATOR_ERROR, AFR_MSG_SELF_HEAL_INFO,
+    AFR_MSG_READ_SUBVOL_ERROR, AFR_MSG_DICT_GET_FAILED, AFR_MSG_INFO_COMMON,
+    AFR_MSG_SPLIT_BRAIN_CHOICE_ERROR, AFR_MSG_LOCAL_CHILD, AFR_MSG_INVALID_DATA,
+    AFR_MSG_INVALID_ARG, AFR_MSG_INDEX_DIR_GET_FAILED, AFR_MSG_FSYNC_FAILED,
+    AFR_MSG_FAVORITE_CHILD, AFR_MSG_SELF_HEAL_FAILED,
+    AFR_MSG_SPLIT_BRAIN_STATUS, AFR_MSG_ADD_BRICK_STATUS, AFR_MSG_NO_CHANGELOG,
+    AFR_MSG_TIMER_CREATE_FAIL, AFR_MSG_SBRAIN_FAV_CHILD_POLICY,
+    AFR_MSG_INODE_CTX_GET_FAILED, AFR_MSG_THIN_ARB,
+    AFR_MSG_THIN_ARB_XATTROP_FAILED, AFR_MSG_THIN_ARB_LOC_POP_FAILED,
+    AFR_MSG_GET_PEND_VAL, AFR_MSG_THIN_ARB_SKIP_SHD, AFR_MSG_UNKNOWN_SET,
+    AFR_MSG_NO_XL_ID, AFR_MSG_SELF_HEAL_INFO_START,
+    AFR_MSG_SELF_HEAL_INFO_FINISH, AFR_MSG_INCRE_COUNT,
+    AFR_MSG_ADD_TO_OUTPUT_FAILED, AFR_MSG_SET_TIME_FAILED,
+    AFR_MSG_GFID_MISMATCH_DETECTED, AFR_MSG_GFID_HEAL_MSG,
+    AFR_MSG_THIN_ARB_LOOKUP_FAILED, AFR_MSG_DICT_CREATE_FAILED,
+    AFR_MSG_NO_MAJORITY_TO_RESOLVE, AFR_MSG_TYPE_MISMATCH,
+    AFR_MSG_SIZE_POLICY_NOT_APPLICABLE, AFR_MSG_NO_CHILD_SELECTED,
+    AFR_MSG_INVALID_CHILD, AFR_MSG_RESOLVE_CONFLICTING_DATA,
+    SERROR_GETTING_SRC_BRICK, SNO_DIFF_IN_MTIME, SNO_BIGGER_FILE,
+    SALL_BRICKS_UP_TO_RESOLVE, AFR_MSG_UNLOCK_FAILED, AFR_MSG_POST_OP_FAILED,
+    AFR_MSG_TA_FRAME_CREATE_FAILED, AFR_MSG_SET_KEY_XATTROP_FAILED,
+    AFR_MSG_BLOCKING_ENTRYLKS_FAILED, AFR_MSG_FOP_FAILED,
+    AFR_MSG_CLEAN_UP_FAILED, AFR_MSG_UNABLE_TO_FETCH, AFR_MSG_XATTR_SET_FAILED,
+    AFR_MSG_SPLIT_BRAIN_REPLICA, AFR_MSG_INODE_CTX_FAILED,
+    AFR_MSG_LOOKUP_FAILED, AFR_MSG_ALL_SUBVOLS_DOWN,
+    AFR_MSG_RELEASE_LOCK_FAILED, AFR_MSG_CLEAR_TIME_SPLIT_BRAIN,
+    AFR_MSG_READ_FAILED, AFR_MSG_LAUNCH_FAILED, AFR_MSG_READ_SUBVOL_NOT_UP,
+    AFR_MSG_LK_HEAL_DOM, AFR_MSG_NEW_BRICK, AFR_MSG_SPLIT_BRAIN_SET_FAILED,
+    AFR_MSG_SPLIT_BRAIN_DETERMINE_FAILED, AFR_MSG_HEALER_SPAWN_FAILED,
+    AFR_MSG_ADD_CRAWL_EVENT_FAILED, AFR_MSG_NULL_DEREF, AFR_MSG_SET_PEND_XATTR,
+    AFR_MSG_INTERNAL_ATTR);
+
+#define AFR_MSG_DICT_GET_FAILED_STR "Dict get failed"
+#define AFR_MSG_DICT_SET_FAILED_STR "Dict set failed"
+#define AFR_MSG_HEALER_SPAWN_FAILED_STR "Healer spawn failed"
+#define AFR_MSG_ADD_CRAWL_EVENT_FAILED_STR "Adding crawl event failed"
+#define AFR_MSG_INVALID_ARG_STR "Invalid argument"
+#define AFR_MSG_INDEX_DIR_GET_FAILED_STR "unable to get index-dir on "
+#define AFR_MSG_THIN_ARB_LOOKUP_FAILED_STR "Failed lookup on file"
+#define AFR_MSG_DICT_CREATE_FAILED_STR "Failed to create dict."
+#define AFR_MSG_THIN_ARB_XATTROP_FAILED_STR "Xattrop failed."
+#define AFR_MSG_THIN_ARB_LOC_POP_FAILED_STR                                    \
+    "Failed to populate loc for thin-arbiter"
+#define AFR_MSG_GET_PEND_VAL_STR "Error getting value of pending"
+#define AFR_MSG_THIN_ARB_SKIP_SHD_STR "I am not the god shd. skipping."
+#define AFR_MSG_UNKNOWN_SET_STR "Unknown set"
+#define AFR_MSG_NO_XL_ID_STR "xl does not have id"
+#define AFR_MSG_SELF_HEAL_INFO_START_STR "starting full sweep on"
+#define AFR_MSG_SELF_HEAL_INFO_FINISH_STR "finished full sweep on"
+#define AFR_MSG_INCRE_COUNT_STR "Could not increment the counter."
+#define AFR_MSG_ADD_TO_OUTPUT_FAILED_STR "Could not add to output"
+#define AFR_MSG_SET_TIME_FAILED_STR "Could not set time"
+#define AFR_MSG_GFID_HEAL_MSG_STR "Error setting gfid-heal-msg dict"
+#define AFR_MSG_NO_MAJORITY_TO_RESOLVE_STR                                     \
+    "No majority to resolve gfid split brain"
+#define AFR_MSG_GFID_MISMATCH_DETECTED_STR "Gfid mismatch dectected"
+#define AFR_MSG_SELF_HEAL_INFO_STR "performing selfheal"
+#define AFR_MSG_TYPE_MISMATCH_STR "TYPE mismatch"
+#define AFR_MSG_SIZE_POLICY_NOT_APPLICABLE_STR                                 \
+    "Size policy is not applicable to directories."
+#define AFR_MSG_NO_CHILD_SELECTED_STR                                          \
+    "No child selected by favorite-child policy"
+#define AFR_MSG_INVALID_CHILD_STR "Invalid child"
+#define AFR_MSG_RESOLVE_CONFLICTING_DATA_STR                                   \
+    "selected as authentic to resolve conflicting data"
+#define SERROR_GETTING_SRC_BRICK_STR "Error getting the source brick"
+#define SNO_DIFF_IN_MTIME_STR "No difference in mtime"
+#define SNO_BIGGER_FILE_STR "No bigger file"
+#define SALL_BRICKS_UP_TO_RESOLVE_STR                                          \
+    "All the bricks should be up to resolve the gfid split brain"
+#define AFR_MSG_UNLOCK_FAILED_STR "Failed to unlock"
+#define AFR_MSG_POST_OP_FAILED_STR "Post-op on thin-arbiter failed"
+#define AFR_MSG_TA_FRAME_CREATE_FAILED_STR "Failed to create ta_frame"
+#define AFR_MSG_SET_KEY_XATTROP_FAILED_STR "Could not set key during xattrop"
+#define AFR_MSG_BLOCKING_ENTRYLKS_FAILED_STR "Blocking entrylks failed"
+#define AFR_MSG_FSYNC_FAILED_STR "fsync failed"
+#define AFR_MSG_QUORUM_FAIL_STR "quorum is not met"
+#define AFR_MSG_FOP_FAILED_STR "Failing Fop"
+#define AFR_MSG_INVALID_SUBVOL_STR "not a subvolume"
+#define AFR_MSG_VOL_MISCONFIGURED_STR "Volume is dangling"
+#define AFR_MSG_CHILD_MISCONFIGURED_STR                                        \
+    "replicate translator needs more than one subvolume defined"
+#define AFR_MSG_CLEAN_UP_FAILED_STR "Failed to clean up healer threads"
+#define AFR_MSG_QUORUM_OVERRIDE_STR "overriding quorum-count"
+#define AFR_MSG_UNABLE_TO_FETCH_STR                                            \
+    "Unable to fetch afr-pending-xattr option from volfile. Falling back to "  \
+    "using client translator names"
+#define AFR_MSG_NULL_DEREF_STR "possible NULL deref"
+#define AFR_MSG_XATTR_SET_FAILED_STR "Cannot set xattr cookie key"
+#define AFR_MSG_SPLIT_BRAIN_STATUS_STR "Failed to create synctask"
+#define AFR_MSG_SUBVOLS_DOWN_STR "All subvolumes are not up"
+#define AFR_MSG_SPLIT_BRAIN_CHOICE_ERROR_STR                                   \
+    "Failed to cancel split-brain choice"
+#define AFR_MSG_SPLIT_BRAIN_REPLICA_STR                                        \
+    "Cannot set replica. File is not in data/metadata split-brain"
+#define AFR_MSG_INODE_CTX_FAILED_STR "Failed to get inode_ctx"
+#define AFR_MSG_READ_SUBVOL_ERROR_STR "no read subvols"
+#define AFR_MSG_LOCAL_CHILD_STR "selecting local read-child"
+#define AFR_MSG_LOOKUP_FAILED_STR "Failed to lookup/create thin-arbiter id file"
+#define AFR_MSG_TIMER_CREATE_FAIL_STR                                          \
+    "Cannot create timer for delayed initialization"
+#define AFR_MSG_SUBVOL_UP_STR "Subvolume came back up; going online"
+#define AFR_MSG_ALL_SUBVOLS_DOWN_STR                                           \
+    "All subvolumes are down. Going offline until atleast one of them is up"
+#define AFR_MSG_RELEASE_LOCK_FAILED_STR "Failed to release lock"
+#define AFR_MSG_INVALID_CHILD_UP_STR "Received child_up from invalid subvolume"
+#define AFR_MSG_QUORUM_MET_STR "Client-quorum is met"
+#define AFR_MSG_EXPUNGING_FILE_OR_DIR_STR "expunging file or dir"
+#define AFR_MSG_SELF_HEAL_FAILED_STR "Invalid"
+#define AFR_MSG_SPLIT_BRAIN_STR "Skipping conservative mergeon the file"
+#define AFR_MSG_CLEAR_TIME_SPLIT_BRAIN_STR "clear time split brain"
+#define AFR_MSG_READ_FAILED_STR "Failing read since good brick is down"
+#define AFR_MSG_LAUNCH_FAILED_STR "Failed to launch synctask"
+#define AFR_MSG_READ_SUBVOL_NOT_UP_STR                                         \
+    "read subvolume in this generation is not up"
+#define AFR_MSG_INTERNAL_LKS_FAILED_STR                                        \
+    "Unable to work with lk-owner while attempting fop"
+#define AFR_MSG_LOCK_XLATOR_NOT_LOADED_STR                                     \
+    "subvolume does not support locking. please load features/locks xlator "   \
+    "on server."
+#define AFR_MSG_FD_CTX_GET_FAILED_STR "unable to get fd ctx"
+#define AFR_MSG_INFO_COMMON_STR "fd not open on any subvolumes, aborting."
+#define AFR_MSG_REPLACE_BRICK_STATUS_STR "Couldn't acquire lock on any child."
+#define AFR_MSG_NEW_BRICK_STR "New brick"
+#define AFR_MSG_SPLIT_BRAIN_SET_FAILED_STR                                     \
+    "Failed to set split-brain choice to -1"
+#define AFR_MSG_SPLIT_BRAIN_DETERMINE_FAILED_STR                               \
+    "Failed to determine split-brain. Aborting split-brain-choice set"
+#define AFR_MSG_OPEN_FAIL_STR "Failed to open subvolume"
+#define AFR_MSG_SET_PEND_XATTR_STR "Set of pending xattr"
+#define AFR_MSG_INTERNAL_ATTR_STR "is an internal extended attribute"
+#endif /* !_AFR_MESSAGES_H_ */
diff --git a/xlators/cluster/afr/src/afr-open.c b/xlators/cluster/afr/src/afr-open.c
index 074a9d32c66..64856042b65 100644
--- a/xlators/cluster/afr/src/afr-open.c
+++ b/xlators/cluster/afr/src/afr-open.c
@@ -1,642 +1,353 @@
 /*
-   Copyright (c) 2007-2010 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
-#include <libgen.h>
 #include <unistd.h>
-#include <fnmatch.h>
 #include <sys/time.h>
 #include <stdlib.h>
 #include <signal.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
+#include <glusterfs/glusterfs.h>
 #include "afr.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "stack.h"
-#include "list.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include "compat-errno.h"
-#include "compat.h"
-#include "byte-order.h"
-#include "statedump.h"
-
-#include "fd.h"
-
-#include "afr-inode-read.h"
-#include "afr-inode-write.h"
-#include "afr-dir-read.h"
-#include "afr-dir-write.h"
-#include "afr-transaction.h"
-#include "afr-self-heal.h"
-#include "afr-self-heal-common.h"
-
-
-int
-afr_open_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this, 
-			int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                        struct iatt *postbuf)
-{
-	afr_local_t * local = frame->local;
-
-	AFR_STACK_UNWIND (open, frame, local->op_ret, local->op_errno,
-			  local->fd);
-	return 0;
-}
+#include <glusterfs/dict.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/byte-order.h>
+#include <glusterfs/statedump.h>
 
+#include "afr-transaction.h"
 
-int
-afr_open_cbk (call_frame_t *frame, void *cookie,
-	      xlator_t *this, int32_t op_ret, int32_t op_errno,
-	      fd_t *fd)
+gf_boolean_t
+afr_is_fd_fixable(fd_t *fd)
 {
-	afr_local_t *  local = NULL;
-
-        int child_index = (long) cookie;
-
-        uint64_t      ctx;
-        afr_fd_ctx_t *fd_ctx;
-
-        int ret = 0;
-
-	int call_count = -1;
-
-	local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret == -1) {
-			local->op_errno = op_errno;
-		}
-
-		if (op_ret >= 0) {
-			local->op_ret = op_ret;
-                        local->success_count++;
-
-                        ret = afr_fd_ctx_set (this, fd);
-
-                        if (ret < 0) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "could not set fd ctx for fd=%p",
-                                        fd);
-
-                                local->op_ret   = -1;
-                                local->op_errno = -ret;
-                                goto unlock;
-                        }
-
-                        ret = fd_ctx_get (fd, this, &ctx);
-
-                        if (ret < 0) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "could not get fd ctx for fd=%p", fd);
-                                local->op_ret   = -1;
-                                local->op_errno = -ret;
-                                goto unlock;
-                        }
-
-                        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
-
-                        fd_ctx->opened_on[child_index] = 1;
-                        fd_ctx->flags                  = local->cont.open.flags;
-                        fd_ctx->wbflags                = local->cont.open.wbflags;
-		}
-	}
-unlock:
-	UNLOCK (&frame->lock);
-
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0) {
-		if ((local->cont.open.flags & O_TRUNC)
-		    && (local->op_ret >= 0)) {
-			STACK_WIND (frame, afr_open_ftruncate_cbk,
-				    this, this->fops->ftruncate,
-				    fd, 0);
-		} else {
-                        AFR_STACK_UNWIND (open, frame, local->op_ret,
-                                          local->op_errno, local->fd);
-		}
-	}
-
-	return 0;
+    if (!fd || !fd->inode)
+        return _gf_false;
+    else if (fd_is_anonymous(fd))
+        return _gf_false;
+    else if (gf_uuid_is_null(fd->inode->gfid))
+        return _gf_false;
+
+    return _gf_true;
 }
 
-
 int
-afr_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-          fd_t *fd, int32_t wbflags)
+afr_open_ftruncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                       struct iatt *postbuf, dict_t *xdata)
 {
-	afr_private_t * priv  = NULL;
-	afr_local_t *   local = NULL;
-
-	int     i = 0;
-	int   ret = -1;
-
-	int32_t call_count = 0;	
-	int32_t op_ret   = -1;
-	int32_t op_errno = 0;
-	int32_t wind_flags = flags & (~O_TRUNC);
-
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
-	VALIDATE_OR_GOTO (loc, out);
-
-	priv = this->private;
-
-        if (afr_is_split_brain (this, loc->inode)) {
-		/* self-heal failed */
-		op_errno = EIO;
-		goto out;
-	}
-
-	ALLOC_OR_GOTO (local, afr_local_t, out);
-
-	ret = AFR_LOCAL_INIT (local, priv);
-	if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-	}
-
-	frame->local = local;
-	call_count   = local->call_count;
-
-        loc_copy (&local->loc, loc);
-
-	local->cont.open.flags   = flags;
-        local->cont.open.wbflags = wbflags;
-
-	local->fd = fd_ref (fd);
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (local->child_up[i]) {
-			STACK_WIND_COOKIE (frame, afr_open_cbk, (void *) (long) i,
-					   priv->children[i],
-					   priv->children[i]->fops->open,
-					   loc, wind_flags, fd, wbflags);
-
-			if (!--call_count)
-				break;
-		}
-	}
+    afr_local_t *local = frame->local;
 
-	op_ret = 0;
-out:
-	if (op_ret == -1) {
-		AFR_STACK_UNWIND (open, frame, op_ret, op_errno, fd);
-	}
-
-	return 0;
+    AFR_STACK_UNWIND(open, frame, local->op_ret, local->op_errno,
+                     local->cont.open.fd, xdata);
+    return 0;
 }
 
-
 int
-afr_openfd_sh_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, fd_t *fd)
+afr_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_private_t       *priv     = NULL;
-
-        int ret = 0;
-
-        uint64_t      ctx;
-        afr_fd_ctx_t *fd_ctx;
-
-        int call_count  = 0;
-        int child_index = (long) cookie;
-
-        priv     = this->private;
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret >= 0) {
-                        ret = fd_ctx_get (fd, this, &ctx);
-
-                        if (ret < 0) {
-                                goto out;
-                        }
-
-                        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
-
-                        fd_ctx->opened_on[child_index] = 1;
-
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "fd for %s opened successfully on subvolume %s",
-                                local->loc.path, priv->children[child_index]->name);
-                }
-        }
-out:
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                int_lock->lock_cbk = local->transaction.done;
-                local->transaction.resume (frame, this);
+    afr_local_t *local = NULL;
+    int call_count = -1;
+    int child_index = (long)cookie;
+    afr_fd_ctx_t *fd_ctx = NULL;
+
+    local = frame->local;
+    fd_ctx = local->fd_ctx;
+
+    local->replies[child_index].valid = 1;
+    local->replies[child_index].op_ret = op_ret;
+    local->replies[child_index].op_errno = op_errno;
+
+    LOCK(&frame->lock);
+    {
+        if (op_ret == -1) {
+            local->op_errno = op_errno;
+            fd_ctx->opened_on[child_index] = AFR_FD_NOT_OPENED;
+        } else {
+            local->op_ret = op_ret;
+            fd_ctx->opened_on[child_index] = AFR_FD_OPENED;
+            if (!local->xdata_rsp && xdata)
+                local->xdata_rsp = dict_ref(xdata);
         }
-
-        return 0;
-}
-
-
-static int
-__unopened_count (int child_count, unsigned int *opened_on, unsigned char *child_up)
-{
-        int i;
-        int count = 0;
-
-        for (i = 0; i < child_count; i++) {
-                if (!opened_on[i] && child_up[i])
-                        count++;
+        call_count = --local->call_count;
+    }
+    UNLOCK(&frame->lock);
+
+    if (call_count == 0) {
+        afr_handle_replies_quorum(frame, this);
+        if (local->op_ret == -1) {
+            AFR_STACK_UNWIND(open, frame, local->op_ret, local->op_errno, NULL,
+                             NULL);
+        } else if (fd_ctx->flags & O_TRUNC) {
+            STACK_WIND(frame, afr_open_ftruncate_cbk, this,
+                       this->fops->ftruncate, fd, 0, NULL);
+        } else {
+            AFR_STACK_UNWIND(open, frame, local->op_ret, local->op_errno,
+                             local->cont.open.fd, local->xdata_rsp);
         }
+    }
 
-        return count;
+    return 0;
 }
 
-
 int
-afr_openfd_sh_unwind (call_frame_t *frame, xlator_t *this)
+afr_open_continue(call_frame_t *frame, xlator_t *this, int err)
 {
-        afr_local_t *local  = NULL;
-        afr_private_t *priv = NULL;
-
-        uint64_t      ctx;
-        afr_fd_ctx_t *fd_ctx;
-
-        int abandon = 0;
-        int ret = 0;
-        int i;
-        int call_count = 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int call_count = 0;
+    int i = 0;
 
-        priv  = this->private;
-        local = frame->local;
+    local = frame->local;
+    priv = this->private;
 
-        /*
-         * Some subvolumes might have come up on which we never
-         * opened this fd in the first place. Re-open fd's on those
-         * subvolumes now.
-         */
-
-        ret = fd_ctx_get (local->fd, this, &ctx);
-
-        if (ret < 0) {
-                abandon = 1;
-                goto out;
-        }
-
-        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
-
-        LOCK (&local->fd->lock);
-        {
-                call_count = __unopened_count (priv->child_count,
-                                               fd_ctx->opened_on,
-                                               local->child_up);
-                for (i = 0; i < priv->child_count; i++) {
-                        fd_ctx->pre_op_done[i] = 0;
-                        fd_ctx->pre_op_piggyback[i] = 0;
-                }
-        }
-        UNLOCK (&local->fd->lock);
-
-        if (call_count == 0) {
-                abandon = 1;
-                goto out;
-        }
-
-        local->call_count = call_count;
+    if (err) {
+        AFR_STACK_UNWIND(open, frame, -1, err, NULL, NULL);
+    } else {
+        local->call_count = AFR_COUNT(local->child_up, priv->child_count);
+        call_count = local->call_count;
 
         for (i = 0; i < priv->child_count; i++) {
-                if (!fd_ctx->opened_on[i] && local->child_up[i]) {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "opening fd for %s on subvolume %s",
-                                local->loc.path, priv->children[i]->name);
-
-                        STACK_WIND_COOKIE (frame, afr_openfd_sh_open_cbk,
-                                           (void *)(long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->open,
-                                           &local->loc, fd_ctx->flags, local->fd,
-                                           fd_ctx->wbflags);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
-
-out:
-        if (abandon)
-                local->transaction.resume (frame, this);
-
-        return 0;
-}
-
-
-static int
-afr_prepare_loc (call_frame_t *frame, fd_t *fd)
-{
-        afr_local_t    *local = NULL;
-        char           *name = NULL;
-        char           *path = NULL;
-        int             ret = 0;
-
-        if (!fd)
-                return -1;
-
-        local = frame->local;
-        ret = inode_path (fd->inode, NULL, (char **)&path);
-        if (ret <= 0)
-                return -1;
-
-        if (local->loc.path) {
-                if (strcmp (path, local->loc.path))
-                        gf_log (frame->this->name, GF_LOG_DEBUG,
-                                "overwriting old loc->path %s with %s",
-                                local->loc.path, path);
-                GF_FREE ((char *)local->loc.path);
-        }
-        local->loc.path = path;
-
-        name = strrchr (local->loc.path, '/');
-        if (name)
-                name++;
-        local->loc.name = name;
-
-        if (local->loc.inode) {
-                inode_unref (local->loc.inode);
+            if (local->child_up[i]) {
+                STACK_WIND_COOKIE(frame, afr_open_cbk, (void *)(long)i,
+                                  priv->children[i],
+                                  priv->children[i]->fops->open, &local->loc,
+                                  (local->cont.open.flags & ~O_TRUNC),
+                                  local->cont.open.fd, local->xdata_req);
+                if (!--call_count)
+                    break;
+            }
         }
-        local->loc.inode = inode_ref (fd->inode);
-
-        if (local->loc.parent) {
-                inode_unref (local->loc.parent);
-        }
-
-        local->loc.parent = inode_parent (local->loc.inode, 0, NULL);
-
-        return 0;
-}
-
-
-int
-afr_openfd_sh (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t     *local  = NULL;
-        afr_self_heal_t *sh = NULL;
-        char            sh_type_str[256] = {0,};
-
-        local = frame->local;
-        sh    = &local->self_heal;
-
-        afr_prepare_loc (frame, local->fd);
-
-        /* forcibly trigger missing-entries self-heal */
-
-        local->success_count    = 1;
-        local->enoent_count     = 1;
-
-        sh->data_lock_held      = _gf_true;
-        sh->need_data_self_heal = _gf_true;
-        sh->type                = local->fd->inode->ia_type;
-        sh->background          = _gf_false;
-        sh->unwind              = afr_openfd_sh_unwind;
-
-        afr_self_heal_type_str_get(&local->self_heal,
-                                   sh_type_str,
-                                   sizeof(sh_type_str));
-        gf_log (this->name, GF_LOG_NORMAL, "%s self-heal triggered. "
-                "path: %s, reason: Replicate up down flush, data lock is held",
-                sh_type_str, local->loc.path);
-
-        afr_self_heal (frame, this);
-
-        return 0;
+    }
+    return 0;
 }
 
-
 int
-afr_openfd_flush_done (call_frame_t *frame, xlator_t *this)
+afr_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+         fd_t *fd, dict_t *xdata)
 {
-        afr_private_t *priv = NULL;
-	afr_local_t *local  = NULL;
-
-        uint64_t       ctx;
-        afr_fd_ctx_t * fd_ctx = NULL;
-
-        int _ret = -1;
-
-        priv  = this->private;
-	local = frame->local;
-
-        LOCK (&local->fd->lock);
-        {
-                _ret = __fd_ctx_get (local->fd, this, &ctx);
-
-                if (_ret < 0) {
-                        goto out;
-                }
-
-                fd_ctx = (afr_fd_ctx_t *)(long) ctx;
-
-                fd_ctx->down_count = priv->down_count;
-                fd_ctx->up_count   = priv->up_count;
-        }
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int spb_subvol = 0;
+    int event_generation = 0;
+    int ret = 0;
+    int32_t op_errno = 0;
+    afr_fd_ctx_t *fd_ctx = NULL;
+
+    // We can't let truncation to happen outside transaction.
+
+    priv = this->private;
+
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
+
+    local->op = GF_FOP_OPEN;
+    fd_ctx = afr_fd_ctx_get(fd, this);
+    if (!fd_ctx) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    if (priv->quorum_count && !afr_has_quorum(local->child_up, this, NULL)) {
+        op_errno = afr_quorum_errno(priv);
+        goto out;
+    }
+
+    if (!afr_is_consistent_io_possible(local, priv, &op_errno))
+        goto out;
+
+    local->inode = inode_ref(loc->inode);
+    loc_copy(&local->loc, loc);
+    local->fd_ctx = fd_ctx;
+    fd_ctx->flags = flags;
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
+
+    local->cont.open.flags = flags;
+    local->cont.open.fd = fd_ref(fd);
+
+    ret = afr_inode_get_readable(frame, local->inode, this, NULL,
+                                 &event_generation, AFR_DATA_TRANSACTION);
+    if ((ret < 0) &&
+        (afr_split_brain_read_subvol_get(local->inode, this, NULL,
+                                         &spb_subvol) == 0) &&
+        spb_subvol < 0) {
+        afr_inode_refresh(frame, this, local->inode, local->inode->gfid,
+                          afr_open_continue);
+    } else {
+        afr_open_continue(frame, this, 0);
+    }
+
+    return 0;
 out:
-        UNLOCK (&local->fd->lock);
-
-        afr_local_transaction_cleanup (local, this);
-
-        gf_log (this->name, GF_LOG_TRACE,
-                "The up/down flush is over");
-
-        fd_unref (local->fd);
-        local->openfd_flush_cbk (frame, this);
+    AFR_STACK_UNWIND(open, frame, -1, op_errno, fd, NULL);
 
-	return 0;
+    return 0;
 }
 
-
-
 int
-afr_openfd_xaction (call_frame_t *frame, xlator_t *this, fd_t *fd)
+afr_openfd_fix_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, fd_t *fd,
+                        dict_t *xdata)
 {
-	afr_local_t   * local = NULL;
-
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
-
-        local = frame->local;
-
-        local->op = GF_FOP_FLUSH;
-
-        local->transaction.fop          = afr_openfd_sh;
-        local->transaction.done         = afr_openfd_flush_done;
-
-        local->transaction.start  = 0;
-        local->transaction.len    = 0;
-
-        gf_log (this->name, GF_LOG_TRACE,
-                "doing up/down flush on fd=%p",
-                fd);
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    afr_fd_ctx_t *fd_ctx = NULL;
+    int call_count = 0;
+    int child_index = (long)cookie;
+
+    priv = this->private;
+    local = frame->local;
+
+    if (op_ret >= 0) {
+        gf_msg_debug(this->name, 0,
+                     "fd for %s opened "
+                     "successfully on subvolume %s",
+                     local->loc.path, priv->children[child_index]->name);
+    } else {
+        gf_smsg(this->name, fop_log_level(GF_FOP_OPEN, op_errno), op_errno,
+                AFR_MSG_OPEN_FAIL, "path=%s", local->loc.path, "subvolume=%s",
+                priv->children[child_index]->name, NULL);
+    }
+
+    fd_ctx = local->fd_ctx;
+
+    LOCK(&local->fd->lock);
+    {
+        if (op_ret >= 0) {
+            fd_ctx->opened_on[child_index] = AFR_FD_OPENED;
+        } else {
+            fd_ctx->opened_on[child_index] = AFR_FD_NOT_OPENED;
+        }
+    }
+    UNLOCK(&local->fd->lock);
 
-        afr_transaction (frame, this, AFR_DATA_TRANSACTION);
+    call_count = afr_frame_return(frame);
+    if (call_count == 0)
+        AFR_STACK_DESTROY(frame);
 
-out:
-	return 0;
+    return 0;
 }
 
-
-
-int
-afr_openfd_xaction_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                             int32_t op_ret, int32_t op_errno, fd_t *fd)
+static int
+afr_fd_ctx_need_open(fd_t *fd, xlator_t *this, unsigned char *need_open)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_private_t       *priv     = NULL;
-
-        int ret = 0;
-
-        uint64_t      ctx = 0;
-        afr_fd_ctx_t *fd_ctx = NULL;
+    afr_fd_ctx_t *fd_ctx = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    int count = 0;
 
-        int call_count  = 0;
-        int child_index = (long) cookie;
+    priv = this->private;
 
-        priv     = this->private;
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret >= 0) {
-                        ret = fd_ctx_get (fd, this, &ctx);
-
-                        if (ret < 0) {
-                                goto out;
-                        }
-
-                        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
-
-                        fd_ctx->opened_on[child_index] = 1;
-
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "fd for %s opened successfully on subvolume %s",
-                                local->loc.path, priv->children[child_index]->name);
-                }
-        }
-out:
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
+    fd_ctx = afr_fd_ctx_get(fd, this);
+    if (!fd_ctx)
+        return 0;
 
-        if (call_count == 0) {
-                afr_openfd_xaction (frame, this, local->fd);
+    LOCK(&fd->lock);
+    {
+        for (i = 0; i < priv->child_count; i++) {
+            if (fd_ctx->opened_on[i] == AFR_FD_NOT_OPENED &&
+                priv->child_up[i]) {
+                fd_ctx->opened_on[i] = AFR_FD_OPENING;
+                need_open[i] = 1;
+                count++;
+            } else {
+                need_open[i] = 0;
+            }
         }
+    }
+    UNLOCK(&fd->lock);
 
-        return 0;
+    return count;
 }
 
-
-int
-afr_openfd_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
+void
+afr_fix_open(fd_t *fd, xlator_t *this)
 {
-        afr_local_t *local  = NULL;
-        afr_private_t *priv = NULL;
-
-        uint64_t      ctx;
-        afr_fd_ctx_t *fd_ctx;
-
-        int no_open = 0;
-        int ret = 0;
-        int i;
-        int call_count = 0;
-
-        priv  = this->private;
-        local = frame->local;
-
-        /*
-         * Some subvolumes might have come up on which we never
-         * opened this fd in the first place. Re-open fd's on those
-         * subvolumes now.
-         */
-
-        local->fd = fd_ref (fd);
-
-        ret = fd_ctx_get (fd, this, &ctx);
-
-        if (ret < 0) {
-                no_open = 1;
-                goto out;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    call_frame_t *frame = NULL;
+    afr_local_t *local = NULL;
+    int ret = -1;
+    int32_t op_errno = 0;
+    afr_fd_ctx_t *fd_ctx = NULL;
+    unsigned char *need_open = NULL;
+    int call_count = 0;
+
+    priv = this->private;
+
+    if (!afr_is_fd_fixable(fd))
+        goto out;
+
+    fd_ctx = afr_fd_ctx_get(fd, this);
+    if (!fd_ctx)
+        goto out;
+
+    need_open = alloca0(priv->child_count);
+
+    call_count = afr_fd_ctx_need_open(fd, this, need_open);
+    if (!call_count)
+        goto out;
+
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame)
+        goto out;
+
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
+
+    local->loc.inode = inode_ref(fd->inode);
+    ret = loc_path(&local->loc, NULL);
+    if (ret < 0)
+        goto out;
+
+    local->fd = fd_ref(fd);
+    local->fd_ctx = fd_ctx;
+
+    local->call_count = call_count;
+
+    gf_msg_debug(this->name, 0, "need open count: %d", call_count);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!need_open[i])
+            continue;
+
+        if (IA_IFDIR == fd->inode->ia_type) {
+            gf_msg_debug(this->name, 0, "opening fd for dir %s on subvolume %s",
+                         local->loc.path, priv->children[i]->name);
+
+            STACK_WIND_COOKIE(frame, afr_openfd_fix_open_cbk, (void *)(long)i,
+                              priv->children[i],
+                              priv->children[i]->fops->opendir, &local->loc,
+                              local->fd, NULL);
+        } else {
+            gf_msg_debug(this->name, 0,
+                         "opening fd for file %s on subvolume %s",
+                         local->loc.path, priv->children[i]->name);
+
+            STACK_WIND_COOKIE(frame, afr_openfd_fix_open_cbk, (void *)(long)i,
+                              priv->children[i], priv->children[i]->fops->open,
+                              &local->loc, fd_ctx->flags & (~O_TRUNC),
+                              local->fd, NULL);
         }
 
-        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
-
-        LOCK (&local->fd->lock);
-        {
-                call_count = __unopened_count (priv->child_count,
-                                               fd_ctx->opened_on,
-                                               local->child_up);
-        }
-        UNLOCK (&local->fd->lock);
-
-        if (call_count == 0) {
-                no_open = 1;
-                goto out;
-        }
-
-        afr_prepare_loc (frame, fd);
-
-        local->call_count = call_count;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (!fd_ctx->opened_on[i] && local->child_up[i]) {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "opening fd for %s on subvolume %s",
-                                local->loc.path, priv->children[i]->name);
-
-                        STACK_WIND_COOKIE (frame, afr_openfd_xaction_open_cbk,
-                                           (void *)(long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->open,
-                                           &local->loc, fd_ctx->flags, fd,
-                                           fd_ctx->wbflags);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
+        if (!--call_count)
+            break;
+    }
 
+    return;
 out:
-        if (no_open)
-                afr_openfd_xaction (frame, this, fd);
-
-        return 0;
+    if (frame)
+        AFR_STACK_DESTROY(frame);
 }
diff --git a/xlators/cluster/afr/src/afr-read-txn.c b/xlators/cluster/afr/src/afr-read-txn.c
new file mode 100644
index 00000000000..6fc2c75145c
--- /dev/null
+++ b/xlators/cluster/afr/src/afr-read-txn.c
@@ -0,0 +1,494 @@
+/*
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include "afr.h"
+#include "afr-transaction.h"
+#include "afr-messages.h"
+
+void
+afr_pending_read_increment(afr_private_t *priv, int child_index)
+{
+    if (child_index < 0 || child_index > priv->child_count)
+        return;
+
+    GF_ATOMIC_INC(priv->pending_reads[child_index]);
+}
+
+void
+afr_pending_read_decrement(afr_private_t *priv, int child_index)
+{
+    if (child_index < 0 || child_index > priv->child_count)
+        return;
+
+    GF_ATOMIC_DEC(priv->pending_reads[child_index]);
+}
+
+void
+afr_read_txn_wind(call_frame_t *frame, xlator_t *this, int subvol)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    afr_pending_read_decrement(priv, local->read_subvol);
+    local->read_subvol = subvol;
+    afr_pending_read_increment(priv, subvol);
+    local->readfn(frame, this, subvol);
+}
+
+int
+afr_read_txn_next_subvol(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    int subvol = -1;
+
+    local = frame->local;
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->readable[i]) {
+            /* don't even bother trying here.
+               just mark as attempted and move on. */
+            local->read_attempted[i] = 1;
+            continue;
+        }
+
+        if (!local->read_attempted[i]) {
+            subvol = i;
+            break;
+        }
+    }
+
+    /* If no more subvols were available for reading, we leave
+       @subvol as -1, which is an indication we have run out of
+       readable subvols. */
+    if (subvol != -1)
+        local->read_attempted[subvol] = 1;
+    afr_read_txn_wind(frame, this, subvol);
+
+    return 0;
+}
+
+static int
+afr_ta_read_txn_done(int ret, call_frame_t *ta_frame, void *opaque)
+{
+    STACK_DESTROY(ta_frame->root);
+    return 0;
+}
+
+static int
+afr_ta_read_txn(void *opaque)
+{
+    call_frame_t *frame = NULL;
+    xlator_t *this = NULL;
+    int read_subvol = -1;
+    int query_child = AFR_CHILD_UNKNOWN;
+    int possible_bad_child = AFR_CHILD_UNKNOWN;
+    int ret = 0;
+    int op_errno = ENOMEM;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    struct gf_flock flock = {
+        0,
+    };
+    dict_t *xdata_req = NULL;
+    dict_t *xdata_rsp = NULL;
+    int **pending = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    frame = (call_frame_t *)opaque;
+    this = frame->this;
+    local = frame->local;
+    priv = this->private;
+    query_child = local->read_txn_query_child;
+
+    if (query_child == AFR_CHILD_ZERO) {
+        possible_bad_child = AFR_CHILD_ONE;
+    } else if (query_child == AFR_CHILD_ONE) {
+        possible_bad_child = AFR_CHILD_ZERO;
+    } else {
+        /*read_txn_query_child is AFR_CHILD_UNKNOWN*/
+        goto out;
+    }
+
+    /* Ask the query_child to see if it blames the possibly bad one. */
+    xdata_req = dict_new();
+    if (!xdata_req)
+        goto out;
+
+    pending = afr_matrix_create(priv->child_count, AFR_NUM_CHANGE_LOGS);
+    if (!pending)
+        goto out;
+
+    ret = afr_set_pending_dict(priv, xdata_req, pending);
+    if (ret < 0)
+        goto out;
+
+    if (local->fd) {
+        ret = syncop_fxattrop(priv->children[query_child], local->fd,
+                              GF_XATTROP_ADD_ARRAY, xdata_req, NULL, &xdata_rsp,
+                              NULL);
+    } else {
+        ret = syncop_xattrop(priv->children[query_child], &local->loc,
+                             GF_XATTROP_ADD_ARRAY, xdata_req, NULL, &xdata_rsp,
+                             NULL);
+    }
+    if (ret || !xdata_rsp) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed xattrop for gfid %s on %s",
+               uuid_utoa(local->inode->gfid),
+               priv->children[query_child]->name);
+        op_errno = -ret;
+        goto out;
+    }
+
+    if (afr_ta_dict_contains_pending_xattr(xdata_rsp, priv,
+                                           possible_bad_child)) {
+        read_subvol = query_child;
+        goto out;
+    }
+    dict_unref(xdata_rsp);
+    xdata_rsp = NULL;
+
+    /* It doesn't. So query thin-arbiter to see if it blames any data brick. */
+    ret = afr_fill_ta_loc(this, &loc, _gf_true);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed to populate thin-arbiter loc for: %s.", loc.name);
+        goto out;
+    }
+    flock.l_type = F_WRLCK; /*start and length are already zero. */
+    ret = syncop_inodelk(priv->children[THIN_ARBITER_BRICK_INDEX],
+                         AFR_TA_DOM_MODIFY, &loc, F_SETLKW, &flock, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "gfid:%s: Failed to get AFR_TA_DOM_MODIFY lock on %s.",
+               uuid_utoa(local->inode->gfid),
+               priv->pending_key[THIN_ARBITER_BRICK_INDEX]);
+        op_errno = -ret;
+        goto out;
+    }
+
+    ret = syncop_xattrop(priv->children[THIN_ARBITER_BRICK_INDEX], &loc,
+                         GF_XATTROP_ADD_ARRAY, xdata_req, NULL, &xdata_rsp,
+                         NULL);
+    if (ret || !xdata_rsp) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "gfid:%s: Failed xattrop on %s.", uuid_utoa(local->inode->gfid),
+               priv->pending_key[THIN_ARBITER_BRICK_INDEX]);
+        op_errno = -ret;
+        goto unlock;
+    }
+
+    if (!afr_ta_dict_contains_pending_xattr(xdata_rsp, priv, query_child)) {
+        read_subvol = query_child;
+    } else {
+        gf_msg(this->name, GF_LOG_ERROR, EIO, AFR_MSG_THIN_ARB,
+               "Failing read for gfid %s since good brick %s is down",
+               uuid_utoa(local->inode->gfid),
+               priv->children[possible_bad_child]->name);
+        op_errno = EIO;
+    }
+
+unlock:
+    flock.l_type = F_UNLCK;
+    ret = syncop_inodelk(priv->children[THIN_ARBITER_BRICK_INDEX],
+                         AFR_TA_DOM_MODIFY, &loc, F_SETLK, &flock, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "gfid:%s: Failed to unlock AFR_TA_DOM_MODIFY lock on "
+               "%s.",
+               uuid_utoa(local->inode->gfid),
+               priv->pending_key[THIN_ARBITER_BRICK_INDEX]);
+    }
+out:
+    if (xdata_req)
+        dict_unref(xdata_req);
+    if (xdata_rsp)
+        dict_unref(xdata_rsp);
+    if (pending)
+        afr_matrix_cleanup(pending, priv->child_count);
+    loc_wipe(&loc);
+
+    if (read_subvol == -1) {
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+    }
+    afr_read_txn_wind(frame, this, read_subvol);
+    return ret;
+}
+
+void
+afr_ta_read_txn_synctask(call_frame_t *frame, xlator_t *this)
+{
+    call_frame_t *ta_frame = NULL;
+    afr_local_t *local = NULL;
+    int ret = 0;
+
+    local = frame->local;
+    ta_frame = afr_ta_frame_create(this);
+    if (!ta_frame) {
+        local->op_ret = -1;
+        local->op_errno = ENOMEM;
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_THIN_ARB,
+               "Failed to create ta_frame");
+        goto out;
+    }
+    ret = synctask_new(this->ctx->env, afr_ta_read_txn, afr_ta_read_txn_done,
+                       ta_frame, frame);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_THIN_ARB,
+               "Failed to launch "
+               "afr_ta_read_txn synctask for gfid %s.",
+               uuid_utoa(local->inode->gfid));
+        local->op_ret = -1;
+        local->op_errno = ENOMEM;
+        STACK_DESTROY(ta_frame->root);
+        goto out;
+    }
+    return;
+out:
+    afr_read_txn_wind(frame, this, -1);
+}
+
+int
+afr_read_txn_refresh_done(call_frame_t *frame, xlator_t *this, int err)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int read_subvol = -1;
+    inode_t *inode = NULL;
+    int ret = -1;
+    int spb_subvol = -1;
+
+    local = frame->local;
+    inode = local->inode;
+    priv = this->private;
+
+    if (err) {
+        if (!priv->thin_arbiter_count)
+            goto readfn;
+        if (err != EINVAL)
+            goto readfn;
+        /* We need to query the good bricks and/or thin-arbiter.*/
+        afr_ta_read_txn_synctask(frame, this);
+        return 0;
+    }
+
+    read_subvol = afr_read_subvol_select_by_policy(inode, this, local->readable,
+                                                   NULL);
+    if (read_subvol == -1) {
+        err = EIO;
+        goto readfn;
+    }
+
+    if (local->read_attempted[read_subvol]) {
+        afr_read_txn_next_subvol(frame, this);
+        return 0;
+    }
+
+    local->read_attempted[read_subvol] = 1;
+readfn:
+    if (read_subvol == -1) {
+        ret = afr_split_brain_read_subvol_get(inode, this, frame, &spb_subvol);
+        if ((ret == 0) && spb_subvol >= 0)
+            read_subvol = spb_subvol;
+    }
+
+    if (read_subvol == -1) {
+        AFR_SET_ERROR_AND_CHECK_SPLIT_BRAIN(-1, err);
+    }
+    afr_read_txn_wind(frame, this, read_subvol);
+
+    return 0;
+}
+
+int
+afr_read_txn_continue(call_frame_t *frame, xlator_t *this, int subvol)
+{
+    afr_local_t *local = NULL;
+
+    local = frame->local;
+
+    if (!local->refreshed) {
+        local->refreshed = _gf_true;
+        afr_inode_refresh(frame, this, local->inode, NULL,
+                          afr_read_txn_refresh_done);
+    } else {
+        afr_read_txn_next_subvol(frame, this);
+    }
+
+    return 0;
+}
+
+/* afr_read_txn_wipe:
+
+   clean internal variables in @local in order to make
+   it possible to call afr_read_txn() multiple times from
+   the same frame
+*/
+
+void
+afr_read_txn_wipe(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+
+    local = frame->local;
+    priv = this->private;
+
+    local->readfn = NULL;
+
+    if (local->inode)
+        inode_unref(local->inode);
+
+    for (i = 0; i < priv->child_count; i++) {
+        local->read_attempted[i] = 0;
+        local->readable[i] = 0;
+    }
+}
+
+/*
+  afr_read_txn:
+
+  This is the read transaction function. The way it works:
+
+  - Determine read-subvolume from inode ctx.
+
+  - If read-subvolume's generation was stale, refresh ctx once by
+    calling afr_inode_refresh()
+
+    Else make an attempt to read on read-subvolume.
+
+  - If attempted read on read-subvolume fails, refresh ctx once
+    by calling afr_inode_refresh()
+
+  - After ctx refresh, query read-subvolume freshly and attempt
+    read once.
+
+  - If read fails, try every other readable[] subvolume before
+    finally giving up. readable[] elements are set by afr_inode_refresh()
+    based on dirty and pending flags.
+
+  - If file is in split brain in the backend, generation will be
+    kept 0 by afr_inode_refresh() and readable[] will be set 0 for
+    all elements. Therefore reads always fail.
+*/
+
+int
+afr_read_txn(call_frame_t *frame, xlator_t *this, inode_t *inode,
+             afr_read_txn_wind_t readfn, afr_transaction_type type)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    unsigned char *data = NULL;
+    unsigned char *metadata = NULL;
+    int read_subvol = -1;
+    int event_generation = 0;
+    int ret = -1;
+
+    priv = this->private;
+    local = frame->local;
+    data = alloca0(priv->child_count);
+    metadata = alloca0(priv->child_count);
+
+    afr_read_txn_wipe(frame, this);
+
+    local->readfn = readfn;
+    local->inode = inode_ref(inode);
+    local->is_read_txn = _gf_true;
+    local->transaction.type = type;
+
+    if (priv->quorum_count && !afr_has_quorum(local->child_up, this, NULL)) {
+        local->op_ret = -1;
+        local->op_errno = afr_quorum_errno(priv);
+        goto read;
+    }
+
+    if (!afr_is_consistent_io_possible(local, priv, &local->op_errno)) {
+        local->op_ret = -1;
+        goto read;
+    }
+
+    if (priv->thin_arbiter_count && !afr_ta_has_quorum(priv, local)) {
+        local->op_ret = -1;
+        local->op_errno = -afr_quorum_errno(priv);
+        goto read;
+    }
+
+    if (priv->thin_arbiter_count &&
+        AFR_COUNT(local->child_up, priv->child_count) != priv->child_count) {
+        if (local->child_up[0]) {
+            local->read_txn_query_child = AFR_CHILD_ZERO;
+        } else if (local->child_up[1]) {
+            local->read_txn_query_child = AFR_CHILD_ONE;
+        }
+        afr_ta_read_txn_synctask(frame, this);
+        return 0;
+    }
+
+    ret = afr_inode_read_subvol_get(inode, this, data, metadata,
+                                    &event_generation);
+    if (ret == -1)
+        /* very first transaction on this inode */
+        goto refresh;
+    AFR_INTERSECT(local->readable, data, metadata, priv->child_count);
+
+    gf_msg_debug(this->name, 0,
+                 "%s: generation now vs cached: %d, "
+                 "%d",
+                 uuid_utoa(inode->gfid), local->event_generation,
+                 event_generation);
+    if (afr_is_inode_refresh_reqd(inode, this, local->event_generation,
+                                  event_generation))
+        /* servers have disconnected / reconnected, and possibly
+           rebooted, very likely changing the state of freshness
+           of copies */
+        goto refresh;
+
+    read_subvol = afr_read_subvol_select_by_policy(inode, this, local->readable,
+                                                   NULL);
+
+    if (read_subvol < 0 || read_subvol > priv->child_count) {
+        gf_msg_debug(this->name, 0,
+                     "Unreadable subvolume %d found "
+                     "with event generation %d for gfid %s.",
+                     read_subvol, event_generation, uuid_utoa(inode->gfid));
+        goto refresh;
+    }
+
+    if (!local->child_up[read_subvol]) {
+        /* should never happen, just in case */
+        gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_READ_SUBVOL_ERROR,
+               "subvolume %d is the "
+               "read subvolume in this generation, but is not up",
+               read_subvol);
+        goto refresh;
+    }
+
+    local->read_attempted[read_subvol] = 1;
+
+read:
+    afr_read_txn_wind(frame, this, read_subvol);
+
+    return 0;
+
+refresh:
+    afr_inode_refresh(frame, this, inode, NULL, afr_read_txn_refresh_done);
+
+    return 0;
+}
diff --git a/xlators/cluster/afr/src/afr-self-heal-algorithm.c b/xlators/cluster/afr/src/afr-self-heal-algorithm.c
deleted file mode 100644
index a65fae0acbf..00000000000
--- a/xlators/cluster/afr/src/afr-self-heal-algorithm.c
+++ /dev/null
@@ -1,1078 +0,0 @@
-/*
-   Copyright (c) 2010 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-
-#include "glusterfs.h"
-#include "afr.h"
-#include "xlator.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "stack.h"
-#include "list.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include "compat-errno.h"
-#include "compat.h"
-#include "byte-order.h"
-#include "md5.h"
-
-#include "afr-transaction.h"
-#include "afr-self-heal.h"
-#include "afr-self-heal-common.h"
-#include "afr-self-heal-algorithm.h"
-
-/*
-  This file contains the various self-heal algorithms
-*/
-
-
-/*
-  The "full" algorithm. Copies the entire file from
-  source to sinks.
-*/
-
-
-static void
-sh_full_private_cleanup (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *               local   = NULL;
-        afr_self_heal_t *           sh      = NULL;
-        afr_sh_algo_full_private_t *sh_priv = NULL;
-
-        local = frame->local;
-        sh    = &local->self_heal;
-
-        sh_priv = sh->private;
-
-        if (sh_priv)
-                GF_FREE (sh_priv);
-}
-
-
-static int
-sh_full_loop_driver (call_frame_t *frame, xlator_t *this, gf_boolean_t is_first_call);
-
-static int
-sh_full_loop_driver_done (call_frame_t *frame, xlator_t *this)
-{
-        afr_private_t * priv = NULL;
-        afr_local_t * local  = NULL;
-        afr_self_heal_t *sh  = NULL;
-        afr_sh_algo_full_private_t *sh_priv = NULL;
-
-        priv    = this->private;
-        local   = frame->local;
-        sh      = &local->self_heal;
-        sh_priv = sh->private;
-
-        sh_full_private_cleanup (frame, this);
-        if (sh->op_failed) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "full self-heal aborting on %s",
-                        local->loc.path);
-
-                local->self_heal.algo_abort_cbk (frame, this);
-        } else {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "full self-heal completed on %s",
-                        local->loc.path);
-
-                local->self_heal.algo_completion_cbk (frame, this);
-        }
-        return 0;
-}
-
-static int
-sh_full_loop_return (call_frame_t *rw_frame, xlator_t *this, off_t offset)
-{
-        afr_local_t *               rw_local   = NULL;
-        afr_self_heal_t *           rw_sh      = NULL;
-
-        call_frame_t *sh_frame              = NULL;
-	afr_local_t * sh_local              = NULL;
-	afr_self_heal_t *sh                 = NULL;
-        afr_sh_algo_full_private_t *sh_priv = NULL;
-
-        rw_local = rw_frame->local;
-        rw_sh    = &rw_local->self_heal;
-
-        sh_frame = rw_sh->sh_frame;
-        sh_local = sh_frame->local;
-        sh       = &sh_local->self_heal;
-        sh_priv  = sh->private;
-
-        AFR_STACK_DESTROY (rw_frame);
-
-        sh_full_loop_driver (sh_frame, this, _gf_false);
-
-        return 0;
-}
-
-
-static int
-sh_full_write_cbk (call_frame_t *rw_frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf)
-{
-	afr_private_t * priv    = NULL;
-	afr_local_t * rw_local  = NULL;
-	afr_self_heal_t *rw_sh  = NULL;
-
-        call_frame_t *sh_frame  = NULL;
-	afr_local_t * sh_local  = NULL;
-	afr_self_heal_t *sh     = NULL;
-
-	int child_index = (long) cookie;
-	int call_count = 0;
-
-	priv = this->private;
-
-	rw_local = rw_frame->local;
-	rw_sh    = &rw_local->self_heal;
-
-        sh_frame = rw_sh->sh_frame;
-        sh_local = sh_frame->local;
-        sh       = &sh_local->self_heal;
-
-	gf_log (this->name, GF_LOG_TRACE,
-		"wrote %d bytes of data from %s to child %d, offset %"PRId64"",
-		op_ret, sh_local->loc.path, child_index,
-                rw_sh->offset - op_ret);
-
-	LOCK (&sh_frame->lock);
-	{
-		if (op_ret == -1) {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"write to %s failed on subvolume %s (%s)",
-				sh_local->loc.path,
-				priv->children[child_index]->name,
-				strerror (op_errno));
-
-			sh->op_failed = 1;
-		}
-	}
-	UNLOCK (&sh_frame->lock);
-
-	call_count = afr_frame_return (rw_frame);
-
-	if (call_count == 0) {
-		sh_full_loop_return (rw_frame, this, rw_sh->offset - op_ret);
-	}
-
-	return 0;
-}
-
-
-static int
-sh_full_read_cbk (call_frame_t *rw_frame, void *cookie,
-                  xlator_t *this, int32_t op_ret, int32_t op_errno,
-                  struct iovec *vector, int32_t count, struct iatt *buf,
-                  struct iobref *iobref)
-{
-	afr_private_t * priv    = NULL;
-	afr_local_t * rw_local  = NULL;
-	afr_self_heal_t *rw_sh  = NULL;
-
-        call_frame_t *sh_frame  = NULL;
-	afr_local_t * sh_local  = NULL;
-	afr_self_heal_t *sh     = NULL;
-
-	int i = 0;
-	int call_count = 0;
-
-	off_t offset = (long) cookie;
-
-	priv = this->private;
-	rw_local = rw_frame->local;
-	rw_sh    = &rw_local->self_heal;
-
-        sh_frame = rw_sh->sh_frame;
-        sh_local = sh_frame->local;
-        sh       = &sh_local->self_heal;
-
-	call_count = sh->active_sinks;
-
-        rw_local->call_count = call_count;
-
-	gf_log (this->name, GF_LOG_TRACE,
-		"read %d bytes of data from %s, offset %"PRId64"",
-		op_ret, sh_local->loc.path, offset);
-
-	if (op_ret <= 0) {
-                sh->op_failed = 1;
-
-                sh_full_loop_return (rw_frame, this, offset);
-		return 0;
-	}
-
-	rw_sh->offset += op_ret;
-
-	if (sh->file_has_holes) {
-		if (iov_0filled (vector, count) == 0) {
-			/* the iter function depends on the
-			   sh->offset already being updated
-			   above
-			*/
-
-                        sh_full_loop_return (rw_frame, this, offset);
-			goto out;
-		}
-	}
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (sh->sources[i] || !sh_local->child_up[i])
-			continue;
-
-		/* this is a sink, so write to it */
-
-		STACK_WIND_COOKIE (rw_frame, sh_full_write_cbk,
-				   (void *) (long) i,
-				   priv->children[i],
-				   priv->children[i]->fops->writev,
-				   sh->healing_fd, vector, count, offset,
-                                   iobref);
-
-		if (!--call_count)
-			break;
-	}
-
-out:
-	return 0;
-}
-
-
-static int
-sh_full_read_write (call_frame_t *frame, xlator_t *this, off_t offset)
-{
-	afr_private_t * priv    = NULL;
-	afr_local_t * local     = NULL;
-	afr_local_t * rw_local  = NULL;
-	afr_self_heal_t *rw_sh  = NULL;
-	afr_self_heal_t *sh     = NULL;
-
-        call_frame_t *rw_frame = NULL;
-
-        int32_t op_errno = 0;
-
-	priv  = this->private;
-        local = frame->local;
-        sh    = &local->self_heal;
-
-        rw_frame = copy_frame (frame);
-        if (!rw_frame)
-                goto out;
-
-        ALLOC_OR_GOTO (rw_local, afr_local_t, out);
-
-        rw_frame->local = rw_local;
-	rw_sh           = &rw_local->self_heal;
-
-        rw_sh->offset       = offset;
-        rw_sh->sh_frame     = frame;
-
-	STACK_WIND_COOKIE (rw_frame, sh_full_read_cbk,
-			   (void *) (long) offset,
-			   priv->children[sh->source],
-			   priv->children[sh->source]->fops->readv,
-			   sh->healing_fd, sh->block_size,
-			   offset);
-        return 0;
-
-out:
-        sh->op_failed = 1;
-
-        sh_full_loop_driver (frame, this, _gf_false);
-
-	return 0;
-}
-
-
-static int
-sh_full_loop_driver (call_frame_t *frame, xlator_t *this, gf_boolean_t is_first_call)
-{
-	afr_private_t * priv = NULL;
-	afr_local_t * local  = NULL;
-	afr_self_heal_t *sh  = NULL;
-        afr_sh_algo_full_private_t *sh_priv = NULL;
-        gf_boolean_t    is_driver_done = _gf_false;
-        blksize_t       block_size = 0;
-        off_t offset  = 0;
-
-        int   loop    = 0;
-
-	priv    = this->private;
-	local   = frame->local;
-	sh      = &local->self_heal;
-        sh_priv = sh->private;
-
-        LOCK (&sh_priv->lock);
-        {
-                if (_gf_false == is_first_call)
-                        sh_priv->loops_running--;
-                offset           = sh_priv->offset;
-                block_size       = sh->block_size;
-                while ((sh->op_failed == 0) &&
-                    (sh_priv->loops_running < priv->data_self_heal_window_size)
-                    && (sh_priv->offset < sh->file_size)) {
-
-                        loop++;
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "spawning a loop for offset %"PRId64,
-                                sh_priv->offset);
-
-                        sh_priv->offset += sh->block_size;
-                        sh_priv->loops_running++;
-
-                        if (_gf_false == is_first_call)
-                                break;
-
-                }
-                if (0 == sh_priv->loops_running) {
-                        is_driver_done = _gf_true;
-                }
-        }
-        UNLOCK (&sh_priv->lock);
-
-        while (loop--) {
-                sh_full_read_write (frame, this, offset);
-                offset += block_size;
-        }
-
-        if (is_driver_done) {
-                sh_full_loop_driver_done (frame, this);
-        }
-
-	return 0;
-}
-
-
-int
-afr_sh_algo_full (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *               local   = NULL;
-        afr_self_heal_t *           sh      = NULL;
-        afr_sh_algo_full_private_t *sh_priv = NULL;
-
-        local = frame->local;
-        sh    = &local->self_heal;
-
-        sh_priv = GF_CALLOC (1, sizeof (*sh_priv),
-                             gf_afr_mt_afr_private_t);
-
-        LOCK_INIT (&sh_priv->lock);
-
-        sh->private = sh_priv;
-
-        local->call_count = 0;
-
-        sh_full_loop_driver (frame, this, _gf_true);
-        return 0;
-}
-
-
-/*
- * The "diff" algorithm. Copies only those blocks whose checksums
- * don't match with those of source.
- */
-
-
-static void
-sh_diff_private_cleanup (call_frame_t *frame, xlator_t *this)
-{
-        afr_private_t *             priv    = NULL;
-        afr_local_t *               local   = NULL;
-        afr_self_heal_t *           sh      = NULL;
-        afr_sh_algo_diff_private_t *sh_priv = NULL;
-
-        int i;
-
-        priv  = this->private;
-        local = frame->local;
-        sh    = &local->self_heal;
-
-        sh_priv = sh->private;
-
-        for (i = 0; i < priv->data_self_heal_window_size; i++) {
-                if (sh_priv->loops[i]) {
-                        if (sh_priv->loops[i]->write_needed)
-                                GF_FREE (sh_priv->loops[i]->write_needed);
-
-                        if (sh_priv->loops[i]->checksum)
-                                GF_FREE (sh_priv->loops[i]->checksum);
-
-                        GF_FREE (sh_priv->loops[i]);
-                }
-        }
-
-        if (sh_priv) {
-                if (sh_priv->loops)
-                        GF_FREE (sh_priv->loops);
-
-                GF_FREE (sh_priv);
-        }
-
-
-}
-
-
-static uint32_t
-__make_cookie (int loop_index, int child_index)
-{
-        uint32_t ret = (loop_index << 16) | child_index;
-        return ret;
-}
-
-
-static int
-__loop_index (uint32_t cookie)
-{
-        return (cookie & 0xFFFF0000) >> 16;
-}
-
-
-static int
-__child_index (uint32_t cookie)
-{
-        return (cookie & 0x0000FFFF);
-}
-
-
-static void
-sh_diff_loop_state_reset (struct sh_diff_loop_state *loop_state, int child_count)
-{
-        loop_state->active = _gf_false;
-//        loop_state->offset = 0;
-
-        memset (loop_state->write_needed,
-                0, sizeof (*loop_state->write_needed) * child_count);
-
-        memset (loop_state->checksum,
-                0, MD5_DIGEST_LEN * child_count);
-}
-
-
-static int
-sh_diff_number_of_writes_needed (unsigned char *write_needed, int child_count)
-{
-        int writes = 0;
-        int i;
-
-        for (i = 0; i < child_count; i++) {
-                if (write_needed[i])
-                        writes++;
-        }
-
-        return writes;
-}
-
-
-static int
-sh_diff_loop_driver_done (call_frame_t *frame, xlator_t *this)
-{
-        afr_private_t *   priv  = NULL;
-        afr_local_t *     local = NULL;
-        afr_self_heal_t * sh    = NULL;
-        afr_sh_algo_diff_private_t *sh_priv = NULL;
-
-
-        priv    = this->private;
-        local   = frame->local;
-        sh      = &local->self_heal;
-        sh_priv = sh->private;
-
-        sh_diff_private_cleanup (frame, this);
-        if (sh->op_failed) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "diff self-heal aborting on %s",
-                        local->loc.path);
-
-                local->self_heal.algo_abort_cbk (frame, this);
-        } else {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "diff self-heal completed on %s",
-                        local->loc.path);
-
-
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "diff self-heal on %s: %d blocks of %d were different (%.2f%%)",
-                        local->loc.path, sh_priv->diff_blocks,
-                        sh_priv->total_blocks,
-                        ((sh_priv->diff_blocks * 1.0)/sh_priv->total_blocks) * 100);
-
-                local->self_heal.algo_completion_cbk (frame, this);
-        }
-
-        return 0;
-}
-
-static int
-sh_diff_loop_driver (call_frame_t *frame, xlator_t *this,
-                     gf_boolean_t is_first_call,
-                     struct sh_diff_loop_state *loop_state);
-
-static int
-sh_diff_loop_return (call_frame_t *rw_frame, xlator_t *this,
-                     struct sh_diff_loop_state *loop_state)
-{
-        afr_private_t *             priv       = NULL;
-        afr_local_t *               rw_local   = NULL;
-        afr_self_heal_t *           rw_sh      = NULL;
-
-        call_frame_t *sh_frame              = NULL;
-	afr_local_t * sh_local              = NULL;
-	afr_self_heal_t *sh                 = NULL;
-        afr_sh_algo_diff_private_t *sh_priv = NULL;
-
-        priv  = this->private;
-
-        rw_local = rw_frame->local;
-        rw_sh    = &rw_local->self_heal;
-
-        sh_frame = rw_sh->sh_frame;
-        sh_local = sh_frame->local;
-        sh       = &sh_local->self_heal;
-        sh_priv  = sh->private;
-
-        gf_log (this->name, GF_LOG_TRACE,
-                "loop for offset %"PRId64" returned", loop_state->offset);
-
-        AFR_STACK_DESTROY (rw_frame);
-
-        sh_diff_loop_driver (sh_frame, this, _gf_false, loop_state);
-
-        return 0;
-}
-
-
-static int
-sh_diff_write_cbk (call_frame_t *rw_frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                   struct iatt *postbuf)
-{
-	afr_private_t *   priv     = NULL;
-	afr_local_t *     rw_local = NULL;
-	afr_self_heal_t * rw_sh    = NULL;
-
-        call_frame_t *sh_frame  = NULL;
-	afr_local_t * sh_local  = NULL;
-	afr_self_heal_t *sh     = NULL;
-
-        afr_sh_algo_diff_private_t *sh_priv;
-        struct sh_diff_loop_state *loop_state;
-
-	int call_count  = 0;
-        int child_index = 0;
-        int loop_index  = 0;
-
-	priv     = this->private;
-	rw_local = rw_frame->local;
-	rw_sh    = &rw_local->self_heal;
-
-        sh_frame = rw_sh->sh_frame;
-        sh_local = sh_frame->local;
-        sh       = &sh_local->self_heal;
-        sh_priv  = sh->private;
-
-        child_index = __child_index ((uint32_t) (long) cookie);
-        loop_index  = __loop_index ((uint32_t) (long) cookie);
-        loop_state  = sh_priv->loops[loop_index];
-
-	gf_log (this->name, GF_LOG_TRACE,
-		"wrote %d bytes of data from %s to child %d, offset %"PRId64"",
-		op_ret, sh_local->loc.path, child_index,
-                loop_state->offset);
-
-	LOCK (&sh_frame->lock);
-	{
-		if (op_ret == -1) {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"write to %s failed on subvolume %s (%s)",
-				sh_local->loc.path,
-				priv->children[child_index]->name,
-				strerror (op_errno));
-
-			sh->op_failed = 1;
-		}
-	}
-	UNLOCK (&sh_frame->lock);
-
-	call_count = afr_frame_return (rw_frame);
-
-	if (call_count == 0) {
-		sh_diff_loop_return (rw_frame, this, loop_state);
-	}
-
-	return 0;
-}
-
-
-static int
-sh_diff_read_cbk (call_frame_t *rw_frame, void *cookie,
-                  xlator_t *this, int32_t op_ret, int32_t op_errno,
-                  struct iovec *vector, int32_t count, struct iatt *buf,
-                  struct iobref *iobref)
-{
-	afr_private_t *   priv     = NULL;
-	afr_local_t *     rw_local = NULL;
-	afr_self_heal_t * rw_sh    = NULL;
-
-        afr_sh_algo_diff_private_t * sh_priv = NULL;
-
-        call_frame_t *sh_frame  = NULL;
-	afr_local_t * sh_local  = NULL;
-	afr_self_heal_t *sh     = NULL;
-
-        int loop_index;
-        struct sh_diff_loop_state *loop_state;
-
-        uint32_t wcookie;
-
-	int i = 0;
-	int call_count = 0;
-
-	priv     = this->private;
-	rw_local = rw_frame->local;
-	rw_sh    = &rw_local->self_heal;
-
-        sh_frame = rw_sh->sh_frame;
-        sh_local = sh_frame->local;
-        sh       = &sh_local->self_heal;
-        sh_priv  = sh->private;
-
-        loop_index = __loop_index ((uint32_t) (long) cookie);
-        loop_state = sh_priv->loops[loop_index];
-
-	call_count = sh_diff_number_of_writes_needed (loop_state->write_needed,
-                                                      priv->child_count);
-
-	rw_local->call_count = call_count;
-
-	gf_log (this->name, GF_LOG_TRACE,
-		"read %d bytes of data from %s, offset %"PRId64"",
-		op_ret, sh_local->loc.path, loop_state->offset);
-
-	if ((op_ret <= 0) ||
-            (call_count == 0)) {
-                sh_diff_loop_return (rw_frame, this, loop_state);
-
-		return 0;
-	}
-
-	if (sh->file_has_holes) {
-		if (iov_0filled (vector, count) == 0) {
-
-                        sh_diff_loop_return (rw_frame, this, loop_state);
-			goto out;
-		}
-	}
-
-	for (i = 0; i < priv->child_count; i++) {
-                if (loop_state->write_needed[i]) {
-                        wcookie = __make_cookie (loop_index, i);
-
-                        STACK_WIND_COOKIE (rw_frame, sh_diff_write_cbk,
-                                           (void *) (long) wcookie,
-                                           priv->children[i],
-                                           priv->children[i]->fops->writev,
-                                           sh->healing_fd, vector, count,
-                                           loop_state->offset, iobref);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
-
-out:
-	return 0;
-}
-
-
-static int
-sh_diff_read (call_frame_t *rw_frame, xlator_t *this,
-              int loop_index)
-{
-	afr_private_t *   priv     = NULL;
-	afr_local_t *     rw_local = NULL;
-	afr_self_heal_t * rw_sh    = NULL;
-
-        afr_sh_algo_diff_private_t * sh_priv = NULL;
-        struct sh_diff_loop_state *loop_state;
-
-        call_frame_t *sh_frame  = NULL;
-	afr_local_t * sh_local  = NULL;
-	afr_self_heal_t *sh     = NULL;
-
-        uint32_t cookie;
-
-	priv     = this->private;
-	rw_local = rw_frame->local;
-	rw_sh    = &rw_local->self_heal;
-
-        sh_frame = rw_sh->sh_frame;
-        sh_local = sh_frame->local;
-        sh       = &sh_local->self_heal;
-        sh_priv  = sh->private;
-
-        loop_state = sh_priv->loops[loop_index];
-
-        cookie = __make_cookie (loop_index, sh->source);
-
-	STACK_WIND_COOKIE (rw_frame, sh_diff_read_cbk,
-			   (void *) (long) cookie,
-			   priv->children[sh->source],
-			   priv->children[sh->source]->fops->readv,
-			   sh->healing_fd, sh_priv->block_size,
-			   loop_state->offset);
-
-	return 0;
-}
-
-
-static int
-sh_diff_checksum_cbk (call_frame_t *rw_frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno,
-                      uint32_t weak_checksum, uint8_t *strong_checksum)
-{
-	afr_private_t * priv    = NULL;
-	afr_local_t * rw_local  = NULL;
-	afr_self_heal_t *rw_sh  = NULL;
-
-        call_frame_t *sh_frame  = NULL;
-	afr_local_t * sh_local  = NULL;
-	afr_self_heal_t *sh     = NULL;
-
-        afr_sh_algo_diff_private_t * sh_priv = NULL;
-
-        int loop_index = 0;
-        int child_index = 0;
-        struct sh_diff_loop_state *loop_state;
-
-        int call_count   = 0;
-        int i            = 0;
-        int write_needed = 0;
-
-	priv  = this->private;
-
-	rw_local = rw_frame->local;
-	rw_sh    = &rw_local->self_heal;
-
-        sh_frame = rw_sh->sh_frame;
-        sh_local = sh_frame->local;
-        sh       = &sh_local->self_heal;
-
-        sh_priv = sh->private;
-
-        child_index = __child_index ((uint32_t) (long) cookie);
-        loop_index  = __loop_index ((uint32_t) (long) cookie);
-
-        loop_state  = sh_priv->loops[loop_index];
-
-        if (op_ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "checksum on %s failed on subvolume %s (%s)",
-                        sh_local->loc.path, priv->children[child_index]->name,
-                        strerror (op_errno));
-
-                sh->op_failed = 1;
-        } else {
-                memcpy (loop_state->checksum + child_index * MD5_DIGEST_LEN,
-                        strong_checksum,
-                        MD5_DIGEST_LEN);
-        }
-
-        call_count = afr_frame_return (rw_frame);
-
-        if (call_count == 0) {
-                for (i = 0; i < priv->child_count; i++) {
-                        if (sh->sources[i] || !sh_local->child_up[i])
-                                continue;
-
-                        if (memcmp (loop_state->checksum + (i * MD5_DIGEST_LEN),
-                                    loop_state->checksum + (sh->source * MD5_DIGEST_LEN),
-                                    MD5_DIGEST_LEN)) {
-                                /*
-                                   Checksums differ, so this block
-                                   must be written to this sink
-                                */
-
-                                gf_log (this->name, GF_LOG_TRACE,
-                                        "checksum on subvolume %s at offset %"
-                                        PRId64" differs from that on source",
-                                        priv->children[i]->name, loop_state->offset);
-
-                                write_needed = loop_state->write_needed[i] = 1;
-                        }
-                }
-
-                LOCK (&sh_priv->lock);
-                {
-                        sh_priv->total_blocks++;
-                        if (write_needed)
-                                sh_priv->diff_blocks++;
-                }
-                UNLOCK (&sh_priv->lock);
-
-                if (write_needed && !sh->op_failed) {
-                        sh_diff_read (rw_frame, this, loop_index);
-                } else {
-                        sh->offset += sh_priv->block_size;
-
-                        sh_diff_loop_return (rw_frame, this, loop_state);
-                }
-        }
-
-        return 0;
-}
-
-
-static int
-sh_diff_find_unused_loop (afr_sh_algo_diff_private_t *sh_priv, int max)
-{
-        int i;
-
-        LOCK (&sh_priv->lock);
-        {
-                for (i = 0; i < max; i++) {
-                        if (sh_priv->loops[i]->active == _gf_false) {
-                                sh_priv->loops[i]->active = _gf_true;
-                                break;
-                        }
-                }
-        }
-        UNLOCK (&sh_priv->lock);
-
-        if (i == max) {
-                gf_log ("[sh-diff]", GF_LOG_ERROR,
-                        "no free loops found! This shouldn't happen. Please"
-                        " report this to gluster-devel@nongnu.org");
-        }
-
-        return i;
-}
-
-
-static int
-sh_diff_checksum (call_frame_t *frame, xlator_t *this, off_t offset)
-{
-	afr_private_t *   priv     = NULL;
-	afr_local_t *     local    = NULL;
-	afr_local_t *     rw_local = NULL;
-	afr_self_heal_t * sh       = NULL;
-	afr_self_heal_t * rw_sh    = NULL;
-
-        afr_sh_algo_diff_private_t * sh_priv = NULL;
-
-        call_frame_t *rw_frame = NULL;
-
-        uint32_t cookie;
-        int loop_index = 0;
-        struct sh_diff_loop_state *loop_state = NULL;
-
-        int32_t op_errno = 0;
-
-        int call_count = 0;
-        int i          = 0;
-
-	priv    = this->private;
-	local   = frame->local;
-	sh      = &local->self_heal;
-
-        sh_priv = sh->private;
-
-        rw_frame = copy_frame (frame);
-        if (!rw_frame)
-                goto out;
-
-        ALLOC_OR_GOTO (rw_local, afr_local_t, out);
-
-        rw_frame->local = rw_local;
-	rw_sh           = &rw_local->self_heal;
-
-        rw_sh->offset       = sh->offset;
-        rw_sh->sh_frame     = frame;
-
-        call_count = sh->active_sinks + 1;  /* sinks and source */
-
-        rw_local->call_count = call_count;
-
-        loop_index = sh_diff_find_unused_loop (sh_priv, priv->data_self_heal_window_size);
-
-        loop_state = sh_priv->loops[loop_index];
-        loop_state->offset       = offset;
-
-        /* we need to send both the loop index and child index,
-           so squeeze them both into a 32-bit number */
-
-        cookie = __make_cookie (loop_index, sh->source);
-
-        STACK_WIND_COOKIE (rw_frame, sh_diff_checksum_cbk,
-                           (void *) (long) cookie,
-                           priv->children[sh->source],
-                           priv->children[sh->source]->fops->rchecksum,
-                           sh->healing_fd,
-                           offset, sh_priv->block_size);
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (sh->sources[i] || !local->child_up[i])
-                        continue;
-
-                cookie = __make_cookie (loop_index, i);
-
-                STACK_WIND_COOKIE (rw_frame, sh_diff_checksum_cbk,
-                                   (void *) (long) cookie,
-                                   priv->children[i],
-                                   priv->children[i]->fops->rchecksum,
-                                   sh->healing_fd,
-                                   offset, sh_priv->block_size);
-
-                if (!--call_count)
-                        break;
-        }
-
-        return 0;
-
-out:
-        sh->op_failed = 1;
-
-        sh_diff_loop_driver (frame, this, _gf_false, loop_state);
-
-        return 0;
-}
-
-
-static int
-sh_diff_loop_driver (call_frame_t *frame, xlator_t *this,
-                     gf_boolean_t is_first_call,
-                     struct sh_diff_loop_state *loop_state)
-{
-	afr_private_t *   priv  = NULL;
-	afr_local_t *     local = NULL;
-	afr_self_heal_t * sh    = NULL;
-        afr_sh_algo_diff_private_t *sh_priv = NULL;
-        gf_boolean_t      is_driver_done = _gf_false;
-        blksize_t         block_size = 0;
-
-        int   loop    = 0;
-
-        off_t offset = 0;
-        char  sh_type_str[256] = {0,};
-
-	priv    = this->private;
-	local   = frame->local;
-	sh      = &local->self_heal;
-        sh_priv = sh->private;
-
-        afr_self_heal_type_str_get(sh, sh_type_str, sizeof(sh_type_str));
-
-        LOCK (&sh_priv->lock);
-        {
-                if (loop_state)
-                        sh_diff_loop_state_reset (loop_state, priv->child_count);
-                if (_gf_false == is_first_call)
-                        sh_priv->loops_running--;
-                offset = sh_priv->offset;
-                block_size = sh_priv->block_size;
-                while ((0 == sh->op_failed) &&
-                    (sh_priv->loops_running < priv->data_self_heal_window_size)
-                    && (sh_priv->offset < sh->file_size)) {
-
-                        loop++;
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "spawning a loop for offset %"PRId64,
-                                sh_priv->offset);
-
-                        sh_priv->offset += sh_priv->block_size;
-                        sh_priv->loops_running++;
-
-                        if (_gf_false == is_first_call)
-                                break;
-
-                }
-                if (0 == sh_priv->loops_running) {
-                        is_driver_done = _gf_true;
-                }
-        }
-        UNLOCK (&sh_priv->lock);
-
-        while (loop--) {
-                sh_diff_checksum (frame, this, offset);
-                offset += block_size;
-        }
-
-        if (is_driver_done) {
-                sh_diff_loop_driver_done (frame, this);
-        }
-        return 0;
-}
-
-
-int
-afr_sh_algo_diff (call_frame_t *frame, xlator_t *this)
-{
-        afr_private_t *             priv    = NULL;
-        afr_local_t *               local   = NULL;
-        afr_self_heal_t *           sh      = NULL;
-        afr_sh_algo_diff_private_t *sh_priv = NULL;
-
-        int i;
-
-        priv  = this->private;
-        local = frame->local;
-        sh    = &local->self_heal;
-
-        sh_priv = GF_CALLOC (1, sizeof (*sh_priv),
-                             gf_afr_mt_afr_private_t);
-
-        sh_priv->block_size = this->ctx->page_size;
-
-        sh->private = sh_priv;
-
-        LOCK_INIT (&sh_priv->lock);
-
-        local->call_count = 0;
-
-        sh_priv->loops = GF_CALLOC (priv->data_self_heal_window_size,
-                                    sizeof (*sh_priv->loops),
-                                    gf_afr_mt_sh_diff_loop_state);
-
-        for (i = 0; i < priv->data_self_heal_window_size; i++) {
-                sh_priv->loops[i]               = GF_CALLOC (1, sizeof (*sh_priv->loops[i]),
-                                                             gf_afr_mt_sh_diff_loop_state);
-
-                sh_priv->loops[i]->checksum     = GF_CALLOC (priv->child_count,
-                                                             MD5_DIGEST_LEN, gf_afr_mt_uint8_t);
-                sh_priv->loops[i]->write_needed = GF_CALLOC (priv->child_count,
-                                                             sizeof (*sh_priv->loops[i]->write_needed),
-                                                             gf_afr_mt_char);
-        }
-
-        sh_diff_loop_driver (frame, this, _gf_true, NULL);
-
-        return 0;
-}
-
-
-struct afr_sh_algorithm afr_self_heal_algorithms[] = {
-        {.name = "full",  .fn = afr_sh_algo_full},
-        {.name = "diff",  .fn = afr_sh_algo_diff},
-        {0, 0},
-};
diff --git a/xlators/cluster/afr/src/afr-self-heal-algorithm.h b/xlators/cluster/afr/src/afr-self-heal-algorithm.h
deleted file mode 100644
index e45621b0ecd..00000000000
--- a/xlators/cluster/afr/src/afr-self-heal-algorithm.h
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
-   Copyright (c) 2010 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef __AFR_SELF_HEAL_ALGORITHM_H__
-#define __AFR_SELF_HEAL_ALGORITHM_H__
-
-
-typedef int (*afr_sh_algo_fn) (call_frame_t *frame,
-                               xlator_t *this);
-
-struct afr_sh_algorithm {
-        const char *name;
-        afr_sh_algo_fn fn;
-};
-
-extern struct afr_sh_algorithm afr_self_heal_algorithms[3];
-
-typedef struct {
-        gf_lock_t lock;
-        unsigned int loops_running;
-        off_t offset;
-} afr_sh_algo_full_private_t;
-
-struct sh_diff_loop_state {
-        off_t   offset;
-        unsigned char *write_needed;
-        uint8_t *checksum;
-        gf_boolean_t active;
-};
-
-typedef struct {
-        size_t block_size;
-
-        gf_lock_t lock;
-        unsigned int loops_running;
-        off_t offset;
-
-        int32_t total_blocks;
-        int32_t diff_blocks;
-
-        struct sh_diff_loop_state **loops;
-} afr_sh_algo_diff_private_t;
-
-#endif /* __AFR_SELF_HEAL_ALGORITHM_H__ */
diff --git a/xlators/cluster/afr/src/afr-self-heal-common.c b/xlators/cluster/afr/src/afr-self-heal-common.c
index 8b0d565f877..a580a1584cc 100644
--- a/xlators/cluster/afr/src/afr-self-heal-common.c
+++ b/xlators/cluster/afr/src/afr-self-heal-common.c
@@ -1,1657 +1,2934 @@
 /*
-  Copyright (c) 2008-2010 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
-#include "glusterfs.h"
-#include "xlator.h"
-#include "byte-order.h"
-
 #include "afr.h"
-#include "afr-transaction.h"
-#include "afr-self-heal-common.h"
 #include "afr-self-heal.h"
-#include "pump.h"
+#include <glusterfs/byte-order.h>
+#include "protocol-common.h"
+#include "afr-messages.h"
+#include <glusterfs/events.h>
 
-/**
- * select_source - select a source and return it
- */
+void
+afr_heal_synctask(xlator_t *this, afr_local_t *local);
 
 int
-afr_sh_select_source (int sources[], int child_count)
+afr_lookup_and_heal_gfid(xlator_t *this, inode_t *parent, const char *name,
+                         inode_t *inode, struct afr_reply *replies, int source,
+                         unsigned char *sources, void *gfid, int *gfid_idx)
 {
-	int i;
-	for (i = 0; i < child_count; i++)
-		if (sources[i])
-			return i;
+    afr_private_t *priv = NULL;
+    call_frame_t *frame = NULL;
+    afr_local_t *local = NULL;
+    unsigned char *wind_on = NULL;
+    ia_type_t ia_type = IA_INVAL;
+    dict_t *xdata = NULL;
+    loc_t loc = {
+        0,
+    };
+    int ret = 0;
+    int i = 0;
+
+    priv = this->private;
+    wind_on = alloca0(priv->child_count);
+    if (source >= 0 && replies[source].valid && replies[source].op_ret == 0)
+        ia_type = replies[source].poststat.ia_type;
+
+    if (ia_type != IA_INVAL)
+        goto heal;
+
+    /* If ia_type is still invalid, it means either
+     * (a)'source' was -1, i.e. parent dir pending xattrs are in split-brain
+     * (or) (b) The parent dir pending xattrs are all zeroes (i.e. all bricks
+     * are sources) and the 'source' we selected earlier might be the one where
+     * the file is not actually present.
+     *
+     * In both cases, let us pick a brick with a successful reply and use its
+     * ia_type.
+     * */
+    for (i = 0; i < priv->child_count; i++) {
+        if (source == -1) {
+            /* case (a) above. */
+            if (replies[i].valid && replies[i].op_ret == 0 &&
+                replies[i].poststat.ia_type != IA_INVAL) {
+                ia_type = replies[i].poststat.ia_type;
+                break;
+            }
+        } else {
+            /* case (b) above. */
+            if (i == source)
+                continue;
+            if (sources[i] && replies[i].valid && replies[i].op_ret == 0 &&
+                replies[i].poststat.ia_type != IA_INVAL) {
+                ia_type = replies[i].poststat.ia_type;
+                break;
+            }
+        }
+    }
 
-	return -1;
-}
+heal:
+    /* gfid heal on those subvolumes that do not have gfid associated
+     * with the inode and update those replies.
+     */
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret != 0)
+            continue;
 
+        if (gf_uuid_is_null(gfid) &&
+            !gf_uuid_is_null(replies[i].poststat.ia_gfid) &&
+            replies[i].poststat.ia_type == ia_type)
+            gfid = replies[i].poststat.ia_gfid;
 
-/**
- * sink_count - return number of sinks in sources array
- */
+        if (!gf_uuid_is_null(replies[i].poststat.ia_gfid) ||
+            replies[i].poststat.ia_type != ia_type)
+            continue;
+
+        wind_on[i] = 1;
+    }
+
+    if (AFR_COUNT(wind_on, priv->child_count) == 0)
+        return 0;
+
+    xdata = dict_new();
+    if (!xdata) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    ret = dict_set_gfuuid(xdata, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    frame = afr_frame_create(this, &ret);
+    if (!frame) {
+        ret = -ret;
+        goto out;
+    }
+
+    local = frame->local;
+    loc.parent = inode_ref(parent);
+    gf_uuid_copy(loc.pargfid, parent->gfid);
+    loc.name = name;
+    loc.inode = inode_ref(inode);
+
+    AFR_ONLIST(wind_on, frame, afr_selfheal_discover_cbk, lookup, &loc, xdata);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!wind_on[i])
+            continue;
+        afr_reply_wipe(&replies[i]);
+        afr_reply_copy(&replies[i], &local->replies[i]);
+    }
+    if (gfid_idx && (*gfid_idx == -1)) {
+        /*Pick a brick where the gifd heal was successful.*/
+        for (i = 0; i < priv->child_count; i++) {
+            if (!wind_on[i])
+                continue;
+            if (replies[i].valid && replies[i].op_ret == 0 &&
+                !gf_uuid_is_null(replies[i].poststat.ia_gfid)) {
+                *gfid_idx = i;
+                break;
+            }
+        }
+    }
+out:
+    if (gfid_idx && (*gfid_idx == -1) && (ret == 0) && local) {
+        ret = -afr_final_errno(local, priv);
+    }
+    loc_wipe(&loc);
+    if (frame)
+        AFR_STACK_DESTROY(frame);
+    if (xdata)
+        dict_unref(xdata);
+
+    return ret;
+}
 
 int
-afr_sh_sink_count (int sources[], int child_count)
+afr_gfid_sbrain_source_from_src_brick(xlator_t *this, struct afr_reply *replies,
+                                      char *src_brick)
 {
-	int i;
-	int sinks = 0;
-	for (i = 0; i < child_count; i++)
-		if (!sources[i])
-			sinks++;
-	return sinks;
+    int i = 0;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret == -1)
+            continue;
+        if (strcmp(priv->children[i]->name, src_brick) == 0)
+            return i;
+    }
+    return -1;
 }
 
 int
-afr_sh_source_count (int sources[], int child_count)
+afr_selfheal_gfid_mismatch_by_majority(struct afr_reply *replies,
+                                       int child_count)
 {
-	int i;
-	int nsource = 0;
+    int j = 0;
+    int i = 0;
+    int votes;
+
+    for (i = 0; i < child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret == -1)
+            continue;
+
+        votes = 1;
+        for (j = i + 1; j < child_count; j++) {
+            if ((!gf_uuid_compare(replies[i].poststat.ia_gfid,
+                                  replies[j].poststat.ia_gfid)))
+                votes++;
+            if (votes > child_count / 2)
+                return i;
+        }
+    }
 
-	for (i = 0; i < child_count; i++)
-		if (sources[i])
-			nsource++;
-	return nsource;
+    return -1;
 }
 
-
 int
-afr_sh_supress_errenous_children (int sources[], int child_errno[],
-				  int child_count)
+afr_gfid_sbrain_source_from_bigger_file(struct afr_reply *replies,
+                                        int child_count)
 {
-	int i = 0;
-
-	for (i = 0; i < child_count; i++) {
-		if (child_errno[i] && sources[i]) {
-			sources[i] = 0;
-		}
-	}
-
-	return 0;
+    int i = 0;
+    int src = -1;
+    uint64_t size = 0;
+
+    for (i = 0; i < child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret == -1)
+            continue;
+        if (size < replies[i].poststat.ia_size) {
+            src = i;
+            size = replies[i].poststat.ia_size;
+        } else if (replies[i].poststat.ia_size == size) {
+            src = -1;
+        }
+    }
+    return src;
 }
 
-
-void
-afr_sh_print_pending_matrix (int32_t *pending_matrix[], xlator_t *this)
+int
+afr_gfid_sbrain_source_from_latest_mtime(struct afr_reply *replies,
+                                         int child_count)
 {
-	afr_private_t * priv = this->private;
-
-	char *buf = NULL;
-	char *ptr = NULL;
-
-	int i, j;
+    int i = 0;
+    int src = -1;
+    uint32_t mtime = 0;
+    uint32_t mtime_nsec = 0;
+
+    for (i = 0; i < child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret != 0)
+            continue;
+        if ((mtime < replies[i].poststat.ia_mtime) ||
+            ((mtime == replies[i].poststat.ia_mtime) &&
+             (mtime_nsec < replies[i].poststat.ia_mtime_nsec))) {
+            src = i;
+            mtime = replies[i].poststat.ia_mtime;
+            mtime_nsec = replies[i].poststat.ia_mtime_nsec;
+        } else if ((mtime == replies[i].poststat.ia_mtime) &&
+                   (mtime_nsec == replies[i].poststat.ia_mtime_nsec)) {
+            src = -1;
+        }
+    }
+    return src;
+}
 
-        /* 10 digits per entry + 1 space + '[' and ']' */
-	buf = GF_MALLOC (priv->child_count * 11 + 8, gf_afr_mt_char);
+int
+afr_gfid_split_brain_source(xlator_t *this, struct afr_reply *replies,
+                            inode_t *inode, uuid_t pargfid, const char *bname,
+                            int src_idx, int child_idx,
+                            unsigned char *locked_on, int *src, dict_t *xdata)
+{
+    afr_private_t *priv = NULL;
+    char g1[64] = {
+        0,
+    };
+    char g2[64] = {
+        0,
+    };
+    int up_count = 0;
+    int heal_op = -1;
+    int ret = -1;
+    char *src_brick = NULL;
+
+    *src = -1;
+    priv = this->private;
+    up_count = AFR_COUNT(locked_on, priv->child_count);
+    if (up_count != priv->child_count) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
+               "All the bricks should be up to resolve the gfid split "
+               "barin");
+        if (xdata) {
+            ret = dict_set_sizen_str_sizen(xdata, "gfid-heal-msg",
+                                           SALL_BRICKS_UP_TO_RESOLVE);
+            if (ret)
+                gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_DICT_SET_FAILED,
+                       "Error setting"
+                       " gfid-heal-msg dict");
+        }
+        goto out;
+    }
 
-	for (i = 0; i < priv->child_count; i++) {
-		ptr = buf;
-		ptr += sprintf (ptr, "[ ");
-		for (j = 0; j < priv->child_count; j++) {
-			ptr += sprintf (ptr, "%d ", pending_matrix[i][j]);
-		}
-		sprintf (ptr, "]");
-		gf_log (this->name, GF_LOG_TRACE,
-			"pending_matrix: %s", buf);
-	}
+    if (xdata) {
+        ret = dict_get_int32_sizen(xdata, "heal-op", &heal_op);
+        if (ret)
+            goto fav_child;
+    } else {
+        goto fav_child;
+    }
+
+    switch (heal_op) {
+        case GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE:
+            *src = afr_gfid_sbrain_source_from_bigger_file(replies,
+                                                           priv->child_count);
+            if (*src == -1) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
+                       SNO_BIGGER_FILE);
+                if (xdata) {
+                    ret = dict_set_sizen_str_sizen(xdata, "gfid-heal-msg",
+                                                   SNO_BIGGER_FILE);
+                    if (ret)
+                        gf_msg(this->name, GF_LOG_ERROR, 0,
+                               AFR_MSG_DICT_SET_FAILED,
+                               "Error"
+                               " setting gfid-heal-msg dict");
+                }
+            }
+            break;
+
+        case GF_SHD_OP_SBRAIN_HEAL_FROM_LATEST_MTIME:
+            *src = afr_gfid_sbrain_source_from_latest_mtime(replies,
+                                                            priv->child_count);
+            if (*src == -1) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
+                       SNO_DIFF_IN_MTIME);
+                if (xdata) {
+                    ret = dict_set_sizen_str_sizen(xdata, "gfid-heal-msg",
+                                                   SNO_DIFF_IN_MTIME);
+                    if (ret)
+                        gf_msg(this->name, GF_LOG_ERROR, 0,
+                               AFR_MSG_DICT_SET_FAILED,
+                               "Error"
+                               "setting gfid-heal-msg dict");
+                }
+            }
+            break;
+
+        case GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK:
+            ret = dict_get_str_sizen(xdata, "child-name", &src_brick);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
+                       "Error getting the source "
+                       "brick");
+                break;
+            }
+            *src = afr_gfid_sbrain_source_from_src_brick(this, replies,
+                                                         src_brick);
+            if (*src == -1) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
+                       SERROR_GETTING_SRC_BRICK);
+                if (xdata) {
+                    ret = dict_set_sizen_str_sizen(xdata, "gfid-heal-msg",
+                                                   SERROR_GETTING_SRC_BRICK);
+                    if (ret)
+                        gf_msg(this->name, GF_LOG_ERROR, 0,
+                               AFR_MSG_DICT_SET_FAILED,
+                               "Error"
+                               " setting gfid-heal-msg dict");
+                }
+            }
+            break;
+
+        default:
+            break;
+    }
+    goto out;
+
+fav_child:
+    switch (priv->fav_child_policy) {
+        case AFR_FAV_CHILD_BY_SIZE:
+            *src = afr_sh_fav_by_size(this, replies, inode);
+            break;
+        case AFR_FAV_CHILD_BY_MTIME:
+            *src = afr_sh_fav_by_mtime(this, replies, inode);
+            break;
+        case AFR_FAV_CHILD_BY_CTIME:
+            *src = afr_sh_fav_by_ctime(this, replies, inode);
+            break;
+        case AFR_FAV_CHILD_BY_MAJORITY:
+            if (priv->child_count != 2)
+                *src = afr_selfheal_gfid_mismatch_by_majority(
+                    replies, priv->child_count);
+            else
+                *src = -1;
+
+            if (*src == -1) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
+                       "No majority to resolve "
+                       "gfid split brain");
+            }
+            break;
+        default:
+            break;
+    }
 
-	GF_FREE (buf);
+out:
+    if (*src == -1) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
+               "Gfid mismatch detected for <gfid:%s>/%s>, %s on %s and"
+               " %s on %s.",
+               uuid_utoa(pargfid), bname,
+               uuid_utoa_r(replies[child_idx].poststat.ia_gfid, g1),
+               priv->children[child_idx]->name,
+               uuid_utoa_r(replies[src_idx].poststat.ia_gfid, g2),
+               priv->children[src_idx]->name);
+        gf_event(EVENT_AFR_SPLIT_BRAIN,
+                 "client-pid=%d;"
+                 "subvol=%s;type=gfid;file="
+                 "<gfid:%s>/%s>;count=2;child-%d=%s;gfid-%d=%s;"
+                 "child-%d=%s;gfid-%d=%s",
+                 this->ctx->cmd_args.client_pid, this->name, uuid_utoa(pargfid),
+                 bname, child_idx, priv->children[child_idx]->name, child_idx,
+                 uuid_utoa_r(replies[child_idx].poststat.ia_gfid, g1), src_idx,
+                 priv->children[src_idx]->name, src_idx,
+                 uuid_utoa_r(replies[src_idx].poststat.ia_gfid, g2));
+        return -1;
+    }
+    return 0;
 }
 
+int
+afr_selfheal_post_op_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
 
-void
-afr_sh_build_pending_matrix (afr_private_t *priv,
-                             int32_t *pending_matrix[], dict_t *xattr[],
-			     int child_count, afr_transaction_type type)
-{
-	int i, j, k;
-
-	/* Indexable by result of afr_index_for_transaction_type(): 0 -- 2. */
-	int32_t pending[3];
-	void *pending_raw = NULL;
-	int ret = -1;
-
-        unsigned char *ignorant_subvols = NULL;
-
-        ignorant_subvols = GF_CALLOC (sizeof (*ignorant_subvols), child_count,
-                                      gf_afr_mt_char);
-
-	/* start clean */
-	for (i = 0; i < child_count; i++) {
-		for (j = 0; j < child_count; j++) {
-			pending_matrix[i][j] = 0;
-		}
-	}
-
-	for (i = 0; i < child_count; i++) {
-		pending_raw = NULL;
-
-                for (j = 0; j < child_count; j++) {
-                        ret = dict_get_ptr (xattr[i], priv->pending_key[j],
-                                            &pending_raw);
-                        
-                        if (ret != 0) {
-                                /*
-                                 * There is no xattr present. This means this
-                                 * subvolume should be considered an 'ignorant'
-                                 * subvolume.
-                                 */
-
-                                ignorant_subvols[i] = 1;
-                                continue;
-                        }
-
-			memcpy (pending, pending_raw, sizeof(pending));
-                        k = afr_index_for_transaction_type (type);
-                        
-                        pending_matrix[i][j] = ntoh32 (pending[k]);
-                }
-	}
-
-        /*
-         * Make all non-ignorant subvols point towards the ignorant
-         * subvolumes.
-         */
+    local = frame->local;
 
-        for (i = 0; i < child_count; i++) {
-                if (ignorant_subvols[i]) {
-                        for (j = 0; j < child_count; j++) {
-                                if (!ignorant_subvols[j])
-                                        pending_matrix[j][i] += 1;
-                        }
-                }
-        }
+    local->op_ret = op_ret;
+    local->op_errno = op_errno;
+    syncbarrier_wake(&local->barrier);
 
-        GF_FREE (ignorant_subvols);
+    return 0;
 }
 
+int
+afr_selfheal_post_op(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                     int subvol, dict_t *xattr, dict_t *xdata)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    loc_t loc = {
+        0,
+    };
+    int ret = 0;
 
-/**
- * mark_sources: Mark all 'source' nodes and return number of source
- * nodes found
- *
- * A node (a row in the pending matrix) belongs to one of
- * three categories:
- *
- * M is the pending matrix.
- *
- * 'innocent' - M[i] is all zeroes
- * 'fool'     - M[i] has i'th element = 1 (self-reference)
- * 'wise'     - M[i] has i'th element = 0, others are 1 or 0.
- *
- * All 'innocent' nodes are sinks. If all nodes are innocent, no self-heal is
- * needed.
- *
- * A 'wise' node can be a source. If two 'wise' nodes conflict, it is 
- * a split-brain. If one wise node refers to the other but the other doesn't
- * refer back, the referrer is a source.
- *
- * All fools are sinks, unless there are no 'wise' nodes. In that case,
- * one of the fools is made a source.
- */
+    priv = this->private;
+    local = frame->local;
 
-typedef enum {
-        AFR_NODE_INNOCENT,
-        AFR_NODE_FOOL,
-        AFR_NODE_WISE
-} afr_node_type;
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
 
-typedef struct {
-        afr_node_type type;
-        int           wisdom;
-} afr_node_character;
+    local->op_ret = 0;
 
+    STACK_WIND(frame, afr_selfheal_post_op_cbk, priv->children[subvol],
+               priv->children[subvol]->fops->xattrop, &loc,
+               GF_XATTROP_ADD_ARRAY, xattr, xdata);
 
-static int
-afr_sh_is_innocent (int32_t *array, int child_count)
-{
-        int i   = 0;
-        int ret = 1;   /* innocent until proven guilty */
+    syncbarrier_wait(&local->barrier, 1);
+    if (local->op_ret < 0)
+        ret = -local->op_errno;
 
-        for (i = 0; i < child_count; i++) {
-                if (array[i]) {
-                        ret = 0;
-                        break;
-                }
-        }
+    loc_wipe(&loc);
+    local->op_ret = 0;
 
-        return ret;
+    return ret;
 }
 
-
-static int
-afr_sh_is_fool (int32_t *array, int i, int child_count)
+int
+afr_check_stale_error(struct afr_reply *replies, afr_private_t *priv)
 {
-        return array[i];   /* fool if accuses itself */
+    int i = 0;
+    int op_errno = 0;
+    int tmp_errno = 0;
+    int stale_count = 0;
+
+    for (i = 0; i < priv->child_count; i++) {
+        tmp_errno = replies[i].op_errno;
+        if (tmp_errno == ENOENT || tmp_errno == ESTALE) {
+            op_errno = afr_higher_errno(op_errno, tmp_errno);
+            stale_count++;
+        }
+    }
+    if (stale_count != priv->child_count)
+        return -ENOTCONN;
+    else
+        return -op_errno;
 }
 
-
-static int
-afr_sh_is_wise (int32_t *array, int i, int child_count)
+int
+afr_sh_generic_fop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno, struct iatt *pre,
+                       struct iatt *post, dict_t *xdata)
 {
-        return !array[i];  /* wise if does not accuse itself */
-}
+    int i = (long)cookie;
+    afr_local_t *local = NULL;
 
+    local = frame->local;
 
-static int
-afr_sh_all_nodes_innocent (afr_node_character *characters, 
-                           int child_count)
-{
-        int i   = 0;
-        int ret = 1;
+    local->replies[i].valid = 1;
+    local->replies[i].op_ret = op_ret;
+    local->replies[i].op_errno = op_errno;
+    if (pre)
+        local->replies[i].prestat = *pre;
+    if (post)
+        local->replies[i].poststat = *post;
+    if (xdata)
+        local->replies[i].xdata = dict_ref(xdata);
 
-        for (i = 0; i < child_count; i++) {
-                if (characters[i].type != AFR_NODE_INNOCENT) {
-                        ret = 0;
-                        break;
-                }
-        }
+    syncbarrier_wake(&local->barrier);
 
-        return ret;
+    return 0;
 }
 
-
-static int
-afr_sh_wise_nodes_exist (afr_node_character *characters, int child_count)
+int
+afr_selfheal_restore_time(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                          int source, unsigned char *healed_sinks,
+                          struct afr_reply *replies)
 {
-        int i   = 0;
-        int ret = 0;
+    loc_t loc = {
+        0,
+    };
 
-        for (i = 0; i < child_count; i++) {
-                if (characters[i].type == AFR_NODE_WISE) {
-                        ret = 1;
-                        break;
-                }
-        }
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
 
-        return ret;
-}
+    AFR_ONLIST(healed_sinks, frame, afr_sh_generic_fop_cbk, setattr, &loc,
+               &replies[source].poststat,
+               (GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME | GF_SET_ATTR_CTIME),
+               NULL);
 
+    loc_wipe(&loc);
 
-/*
- * The 'wisdom' of a wise node is 0 if any other wise node accuses it.
- * It is 1 if no other wise node accuses it. 
- * Only wise nodes with wisdom 1 are sources.
- *
- * If no nodes with wisdom 1 exist, a split-brain has occured.
- */
-
-static void
-afr_sh_compute_wisdom (int32_t *pending_matrix[],
-                       afr_node_character characters[], int child_count)
-{
-        int i = 0;
-        int j = 0;
-
-        for (i = 0; i < child_count; i++) {
-                if (characters[i].type == AFR_NODE_WISE) {
-                        characters[i].wisdom = 1;
-
-                        for (j = 0; j < child_count; j++) {
-                                if ((characters[j].type == AFR_NODE_WISE)
-                                    && pending_matrix[j][i]) {
-                                        
-                                        characters[i].wisdom = 0;
-                                }
-                        }
-                }
-        }
+    return 0;
 }
 
-
-static int
-afr_sh_wise_nodes_conflict (afr_node_character *characters, 
-                            int child_count)
+dict_t *
+afr_selfheal_output_xattr(xlator_t *this, gf_boolean_t is_full_crawl,
+                          afr_transaction_type type, int *output_dirty,
+                          int **output_matrix, int subvol,
+                          int **full_heal_mtx_out)
 {
-        int i   = 0;
-        int ret = 1;
-
-        for (i = 0; i < child_count; i++) {
-                if ((characters[i].type == AFR_NODE_WISE)
-                    && characters[i].wisdom == 1) {
-
-                        /* There is atleast one bona-fide wise node */
-                        ret = 0;
-                        break;
-                }
+    int j = 0;
+    int idx = 0;
+    int d_idx = 0;
+    int ret = 0;
+    int *raw = 0;
+    dict_t *xattr = NULL;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+    idx = afr_index_for_transaction_type(type);
+    d_idx = afr_index_for_transaction_type(AFR_DATA_TRANSACTION);
+
+    xattr = dict_new();
+    if (!xattr)
+        return NULL;
+
+    /* clear dirty */
+    raw = GF_CALLOC(sizeof(int), AFR_NUM_CHANGE_LOGS, gf_afr_mt_int32_t);
+    if (!raw)
+        goto err;
+
+    raw[idx] = hton32(output_dirty[subvol]);
+    ret = dict_set_bin(xattr, AFR_DIRTY, raw,
+                       sizeof(int) * AFR_NUM_CHANGE_LOGS);
+    if (ret) {
+        GF_FREE(raw);
+        goto err;
+    }
+
+    /* clear/set pending */
+    for (j = 0; j < priv->child_count; j++) {
+        raw = GF_CALLOC(sizeof(int), AFR_NUM_CHANGE_LOGS, gf_afr_mt_int32_t);
+        if (!raw)
+            goto err;
+
+        raw[idx] = hton32(output_matrix[subvol][j]);
+        if (is_full_crawl)
+            raw[d_idx] = hton32(full_heal_mtx_out[subvol][j]);
+
+        ret = dict_set_bin(xattr, priv->pending_key[j], raw,
+                           sizeof(int) * AFR_NUM_CHANGE_LOGS);
+        if (ret) {
+            GF_FREE(raw);
+            goto err;
         }
+    }
 
-        return ret;
+    return xattr;
+err:
+    if (xattr)
+        dict_unref(xattr);
+    return NULL;
 }
 
+int
+afr_selfheal_undo_pending(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                          unsigned char *sources, unsigned char *sinks,
+                          unsigned char *healed_sinks,
+                          unsigned char *undid_pending,
+                          afr_transaction_type type, struct afr_reply *replies,
+                          unsigned char *locked_on)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int i = 0;
+    int j = 0;
+    unsigned char *pending = NULL;
+    int *input_dirty = NULL;
+    int **input_matrix = NULL;
+    int **full_heal_mtx_in = NULL;
+    int **full_heal_mtx_out = NULL;
+    int *output_dirty = NULL;
+    int **output_matrix = NULL;
+    dict_t *xattr = NULL;
+    dict_t *xdata = NULL;
+
+    priv = this->private;
+    local = frame->local;
+
+    pending = alloca0(priv->child_count);
+
+    input_dirty = alloca0(priv->child_count * sizeof(int));
+    input_matrix = ALLOC_MATRIX(priv->child_count, int);
+    full_heal_mtx_in = ALLOC_MATRIX(priv->child_count, int);
+    full_heal_mtx_out = ALLOC_MATRIX(priv->child_count, int);
+    output_dirty = alloca0(priv->child_count * sizeof(int));
+    output_matrix = ALLOC_MATRIX(priv->child_count, int);
+
+    xdata = dict_new();
+    if (!xdata)
+        return -1;
+
+    afr_selfheal_extract_xattr(this, replies, type, input_dirty, input_matrix);
+
+    if (local->need_full_crawl)
+        afr_selfheal_extract_xattr(this, replies, AFR_DATA_TRANSACTION, NULL,
+                                   full_heal_mtx_in);
+
+    for (i = 0; i < priv->child_count; i++)
+        if (sinks[i] && !healed_sinks[i])
+            pending[i] = 1;
+
+    for (i = 0; i < priv->child_count; i++) {
+        for (j = 0; j < priv->child_count; j++) {
+            if (pending[j]) {
+                output_matrix[i][j] = 1;
+                if (type == AFR_ENTRY_TRANSACTION)
+                    full_heal_mtx_out[i][j] = 1;
+            } else if (locked_on[j]) {
+                output_matrix[i][j] = -input_matrix[i][j];
+                if (type == AFR_ENTRY_TRANSACTION)
+                    full_heal_mtx_out[i][j] = -full_heal_mtx_in[i][j];
+            }
+        }
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!pending[i])
+            output_dirty[i] = -input_dirty[i];
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!locked_on[i])
+            /* perform post-op only on subvols we had locked
+               and inspected on.
+            */
+            continue;
+        if (undid_pending[i])
+            /* We already unset the pending xattrs in
+             * _afr_fav_child_reset_sink_xattrs(). */
+            continue;
+
+        xattr = afr_selfheal_output_xattr(this, local->need_full_crawl, type,
+                                          output_dirty, output_matrix, i,
+                                          full_heal_mtx_out);
+        if (!xattr) {
+            continue;
+        }
 
-static int
-afr_sh_mark_wisest_as_sources (int sources[], 
-                               afr_node_character *characters, 
-                               int child_count)
-{
-        int nsources = 0;
-        
-        int i = 0;
-
-        for (i = 0; i < child_count; i++) {
-                if (characters[i].wisdom == 1) {
-                        sources[i] = 1;
-                        nsources++;
-                }
+        if ((type == AFR_ENTRY_TRANSACTION) && (priv->esh_granular)) {
+            if (xdata && dict_set_int8(xdata, GF_XATTROP_PURGE_INDEX, 1))
+                gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_DICT_SET_FAILED,
+                       "Failed to set"
+                       " dict value for %s",
+                       GF_XATTROP_PURGE_INDEX);
         }
 
-        return nsources;
+        afr_selfheal_post_op(frame, this, inode, i, xattr, xdata);
+        dict_unref(xattr);
+    }
+
+    if (xdata)
+        dict_unref(xdata);
+
+    return 0;
 }
 
+void
+afr_reply_copy(struct afr_reply *dst, struct afr_reply *src)
+{
+    dict_t *xdata = NULL;
+
+    dst->valid = src->valid;
+    dst->op_ret = src->op_ret;
+    dst->op_errno = src->op_errno;
+    dst->prestat = src->prestat;
+    dst->poststat = src->poststat;
+    dst->preparent = src->preparent;
+    dst->postparent = src->postparent;
+    dst->preparent2 = src->preparent2;
+    dst->postparent2 = src->postparent2;
+    if (src->xdata)
+        xdata = dict_ref(src->xdata);
+    else
+        xdata = NULL;
+    if (dst->xdata)
+        dict_unref(dst->xdata);
+    dst->xdata = xdata;
+    if (xdata && dict_get_str_boolean(xdata, "fips-mode-rchecksum",
+                                      _gf_false) == _gf_true) {
+        memcpy(dst->checksum, src->checksum, SHA256_DIGEST_LENGTH);
+    } else {
+        memcpy(dst->checksum, src->checksum, MD5_DIGEST_LENGTH);
+    }
+    dst->fips_mode_rchecksum = src->fips_mode_rchecksum;
+}
 
-static int
-afr_sh_mark_if_size_differs (afr_self_heal_t *sh, int child_count)
+void
+afr_replies_copy(struct afr_reply *dst, struct afr_reply *src, int count)
 {
-        int32_t ** pending_matrix;
-        int i, j;
+    int i = 0;
 
-        int size_differs = 0;
+    if (dst == src)
+        return;
 
-        pending_matrix = sh->pending_matrix;
+    for (i = 0; i < count; i++) {
+        afr_reply_copy(&dst[i], &src[i]);
+    }
+}
 
-        for (i = 0; i < child_count; i++) {
-                for (j = 0; j < child_count; j++) {
-                        if (!sh->buf)
-                                break;
+int
+afr_selfheal_fill_dirty(xlator_t *this, int *dirty, int subvol, int idx,
+                        dict_t *xdata)
+{
+    void *pending_raw = NULL;
+    int pending[3] = {
+        0,
+    };
 
-                        if (SIZE_DIFFERS (&sh->buf[i], &sh->buf[j])
-                            && (pending_matrix[i][j] == 0)
-                            && (pending_matrix[j][i] == 0)) {
+    if (!dirty)
+        return 0;
 
-                                pending_matrix[i][j] = 1;
-                                pending_matrix[j][i] = 1;
+    if (dict_get_ptr(xdata, AFR_DIRTY, &pending_raw))
+        return -1;
 
-                                size_differs = 1;
-                        }
-                }
-        }
+    if (!pending_raw)
+        return -1;
 
-        return size_differs;
+    memcpy(pending, pending_raw, sizeof(pending));
+
+    dirty[subvol] = ntoh32(pending[idx]);
+
+    return 0;
 }
 
-        
-static int
-afr_sh_mark_biggest_fool_as_source (afr_self_heal_t *sh,
-                                    afr_node_character *characters, 
-                                    int child_count)
+int
+afr_selfheal_fill_matrix(xlator_t *this, int **matrix, int subvol, int idx,
+                         dict_t *xdata)
 {
-        int i = 0;
-        int biggest = 0;
+    int i = 0;
+    void *pending_raw = NULL;
+    int pending[3] = {
+        0,
+    };
+    afr_private_t *priv = NULL;
 
-        for (i = 0; i < child_count; i++) {
-                if (characters[i].type == AFR_NODE_FOOL) {
-                        biggest = i;
-                        break;
-                }
-        }
+    priv = this->private;
 
-        for (i = 0; i < child_count; i++) {
-                if (characters[i].type != AFR_NODE_FOOL)
-                        continue;
+    if (!matrix)
+        return 0;
 
-                if (!sh->buf)
-                        break;
+    for (i = 0; i < priv->child_count; i++) {
+        if (dict_get_ptr(xdata, priv->pending_key[i], &pending_raw))
+            continue;
 
-                if (SIZE_GREATER (&sh->buf[i], &sh->buf[biggest])) {
-                        biggest = i;
-                }
-        }
+        if (!pending_raw)
+            continue;
 
-        sh->sources[biggest] = 1;
+        memcpy(pending, pending_raw, sizeof(pending));
 
-        return 1;
-}
+        matrix[subvol][i] = ntoh32(pending[idx]);
+    }
 
+    return 0;
+}
 
-static int
-afr_sh_mark_biggest_as_source (afr_self_heal_t *sh, int child_count)
+int
+afr_selfheal_extract_xattr(xlator_t *this, struct afr_reply *replies,
+                           afr_transaction_type type, int *dirty, int **matrix)
 {
-        int biggest = 0;
-        int i;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    dict_t *xdata = NULL;
+    int idx = -1;
 
-        for (i = 0; i < child_count; i++) {
-                if (!sh->buf)
-                        break;
+    idx = afr_index_for_transaction_type(type);
 
-                if (SIZE_GREATER (&sh->buf[i], &sh->buf[biggest])) {
-                        biggest = i;
-                }
-        }
+    priv = this->private;
 
-        sh->sources[biggest] = 1;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret != 0)
+            continue;
 
-        return 1;
-}
+        if (!replies[i].xdata)
+            continue;
 
+        xdata = replies[i].xdata;
 
-static int
-afr_sh_mark_loweia_uid_as_source (afr_self_heal_t *sh, int child_count)
-{
-        uid_t smallest = 0;
-        int i;
+        afr_selfheal_fill_dirty(this, dirty, i, idx, xdata);
+        afr_selfheal_fill_matrix(this, matrix, i, idx, xdata);
+    }
 
-        for (i = 0; i < child_count; i++) {
-                if (!sh->buf)
-                        break;
+    return 0;
+}
 
-                if (sh->buf[i].ia_uid < sh->buf[smallest].ia_uid) {
-                        smallest = i;
-                }
+/*
+ * If by chance there are multiple sources with differing sizes, select
+ * the largest file as the source.
+ *
+ * This can happen if data was directly modified in the backend or for snapshots
+ */
+void
+afr_mark_largest_file_as_source(xlator_t *this, unsigned char *sources,
+                                struct afr_reply *replies)
+{
+    int i = 0;
+    afr_private_t *priv = NULL;
+    uint64_t size = 0;
+
+    /* Find source with biggest file size */
+    priv = this->private;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i])
+            continue;
+        if (!replies[i].valid || replies[i].op_ret != 0) {
+            sources[i] = 0;
+            continue;
         }
-
-        sh->sources[smallest] = 1;
-
-        return 1;
+        if (size <= replies[i].poststat.ia_size) {
+            size = replies[i].poststat.ia_size;
+        }
+    }
+
+    /* Mark sources with less size as not source */
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i])
+            continue;
+        if (size > replies[i].poststat.ia_size)
+            sources[i] = 0;
+    }
 }
 
-
-int
-afr_sh_mark_sources (afr_self_heal_t *sh, int child_count,
-                     afr_self_heal_type type)
+void
+afr_mark_latest_mtime_file_as_source(xlator_t *this, unsigned char *sources,
+                                     struct afr_reply *replies)
 {
-	int i = 0;
+    int i = 0;
+    afr_private_t *priv = NULL;
+    uint32_t mtime = 0;
+    uint32_t mtime_nsec = 0;
+
+    priv = this->private;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i])
+            continue;
+        if (!replies[i].valid || replies[i].op_ret != 0) {
+            sources[i] = 0;
+            continue;
+        }
+        if ((mtime < replies[i].poststat.ia_mtime) ||
+            ((mtime == replies[i].poststat.ia_mtime) &&
+             (mtime_nsec < replies[i].poststat.ia_mtime_nsec))) {
+            mtime = replies[i].poststat.ia_mtime;
+            mtime_nsec = replies[i].poststat.ia_mtime_nsec;
+        }
+    }
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i])
+            continue;
+        if ((mtime > replies[i].poststat.ia_mtime) ||
+            ((mtime == replies[i].poststat.ia_mtime) &&
+             (mtime_nsec > replies[i].poststat.ia_mtime_nsec))) {
+            sources[i] = 0;
+        }
+    }
+}
 
-        int32_t ** pending_matrix;
-        int *      sources;
+void
+afr_mark_active_sinks(xlator_t *this, unsigned char *sources,
+                      unsigned char *locked_on, unsigned char *sinks)
+{
+    int i = 0;
+    afr_private_t *priv = NULL;
 
-        int size_differs = 0;
+    priv = this->private;
 
-        pending_matrix = sh->pending_matrix;
-        sources        = sh->sources;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i] && locked_on[i])
+            sinks[i] = 1;
+        else
+            sinks[i] = 0;
+    }
+}
 
-	int nsources = 0;
+gf_boolean_t
+afr_dict_contains_heal_op(call_frame_t *frame)
+{
+    afr_local_t *local = NULL;
+    dict_t *xdata_req = NULL;
+    int ret = 0;
+    int heal_op = -1;
+
+    local = frame->local;
+    xdata_req = local->xdata_req;
+    ret = dict_get_int32_sizen(xdata_req, "heal-op", &heal_op);
+    if (ret)
+        return _gf_false;
+    if (local->xdata_rsp == NULL) {
+        local->xdata_rsp = dict_new();
+        if (!local->xdata_rsp)
+            return _gf_true;
+    }
+    ret = dict_set_sizen_str_sizen(local->xdata_rsp, "sh-fail-msg",
+                                   SFILE_NOT_IN_SPLIT_BRAIN);
+
+    return _gf_true;
+}
 
-        /* stores the 'characters' (innocent, fool, wise) of the nodes */
-        afr_node_character *
-                characters = GF_CALLOC (sizeof (afr_node_character), 
-                                        child_count,
-                                        gf_afr_mt_afr_node_character) ;
+gf_boolean_t
+afr_can_decide_split_brain_source_sinks(struct afr_reply *replies,
+                                        int child_count)
+{
+    int i = 0;
 
-	/* start clean */
-	for (i = 0; i < child_count; i++) {
-		sources[i] = 0;
-	}
-        
-        for (i = 0; i < child_count; i++) {
-                if (afr_sh_is_innocent (pending_matrix[i], child_count)) {
-                        characters[i].type = AFR_NODE_INNOCENT;
+    for (i = 0; i < child_count; i++)
+        if (replies[i].valid != 1 || replies[i].op_ret != 0)
+            return _gf_false;
 
-                } else if (afr_sh_is_fool (pending_matrix[i], i, child_count)) {
-                        characters[i].type = AFR_NODE_FOOL;
+    return _gf_true;
+}
 
-                } else if (afr_sh_is_wise (pending_matrix[i], i, child_count)) {
-                        characters[i].type = AFR_NODE_WISE;
+int
+afr_mark_split_brain_source_sinks_by_heal_op(
+    call_frame_t *frame, xlator_t *this, unsigned char *sources,
+    unsigned char *sinks, unsigned char *healed_sinks, unsigned char *locked_on,
+    struct afr_reply *replies, afr_transaction_type type, int heal_op)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    dict_t *xdata_req = NULL;
+    dict_t *xdata_rsp = NULL;
+    int ret = 0;
+    int i = 0;
+    char *name = NULL;
+    int source = -1;
+
+    local = frame->local;
+    priv = this->private;
+    xdata_req = local->xdata_req;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (locked_on[i])
+            if (sources[i] || !sinks[i] || !healed_sinks[i]) {
+                ret = -1;
+                goto out;
+            }
+    }
+    if (local->xdata_rsp == NULL) {
+        local->xdata_rsp = dict_new();
+        if (!local->xdata_rsp) {
+            ret = -1;
+            goto out;
+        }
+    }
+    xdata_rsp = local->xdata_rsp;
+
+    if (!afr_can_decide_split_brain_source_sinks(replies, priv->child_count)) {
+        ret = dict_set_sizen_str_sizen(xdata_rsp, "sh-fail-msg",
+                                       SBRAIN_HEAL_NO_GO_MSG);
+        ret = -1;
+        goto out;
+    }
+
+    for (i = 0; i < priv->child_count; i++)
+        if (locked_on[i])
+            sources[i] = 1;
+    switch (heal_op) {
+        case GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE:
+            if (type == AFR_METADATA_TRANSACTION) {
+                ret = dict_set_sizen_str_sizen(xdata_rsp, "sh-fail-msg",
+                                               SUSE_SOURCE_BRICK_TO_HEAL);
+                if (!ret)
+                    ret = -1;
+                goto out;
+            }
+            afr_mark_largest_file_as_source(this, sources, replies);
+            if (AFR_COUNT(sources, priv->child_count) != 1) {
+                ret = dict_set_sizen_str_sizen(xdata_rsp, "sh-fail-msg",
+                                               SNO_BIGGER_FILE);
+                if (!ret)
+                    ret = -1;
+                goto out;
+            }
+            break;
+        case GF_SHD_OP_SBRAIN_HEAL_FROM_LATEST_MTIME:
+            if (type == AFR_METADATA_TRANSACTION) {
+                ret = dict_set_sizen_str_sizen(xdata_rsp, "sh-fail-msg",
+                                               SUSE_SOURCE_BRICK_TO_HEAL);
+                if (!ret)
+                    ret = -1;
+                goto out;
+            }
+            afr_mark_latest_mtime_file_as_source(this, sources, replies);
+            if (AFR_COUNT(sources, priv->child_count) != 1) {
+                ret = dict_set_sizen_str_sizen(xdata_rsp, "sh-fail-msg",
+                                               SNO_DIFF_IN_MTIME);
+                if (!ret)
+                    ret = -1;
+                goto out;
+            }
+            break;
+        case GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK:
+            ret = dict_get_str_sizen(xdata_req, "child-name", &name);
+            if (ret)
+                goto out;
+            source = afr_get_child_index_from_name(this, name);
+            if (source < 0) {
+                ret = dict_set_sizen_str_sizen(xdata_rsp, "sh-fail-msg",
+                                               SINVALID_BRICK_NAME);
+                if (!ret)
+                    ret = -1;
+                goto out;
+            }
+            if (locked_on[source] != 1) {
+                ret = dict_set_sizen_str_sizen(xdata_rsp, "sh-fail-msg",
+                                               SBRICK_IS_NOT_UP);
+                if (!ret)
+                    ret = -1;
+                goto out;
+            }
+            memset(sources, 0, sizeof(*sources) * priv->child_count);
+            sources[source] = 1;
+            break;
+        default:
+            ret = -1;
+            goto out;
+    }
+    for (i = 0; i < priv->child_count; i++) {
+        if (sources[i]) {
+            source = i;
+            break;
+        }
+    }
+    sinks[source] = 0;
+    healed_sinks[source] = 0;
+    ret = source;
+out:
+    if (ret < 0)
+        memset(sources, 0, sizeof(*sources) * priv->child_count);
+    return ret;
+}
 
-                } else {
-                        gf_log ("[module:replicate]", GF_LOG_ERROR,
-                                "Could not determine the state of subvolume %d!"
-                                " (This message should never appear."
-                                " Please file a bug report to "
-                                "<gluster-devel@nongnu.org>.)", i);
+int
+afr_sh_fav_by_majority(xlator_t *this, struct afr_reply *replies,
+                       inode_t *inode)
+{
+    afr_private_t *priv;
+    int vote_count = -1;
+    int fav_child = -1;
+    int i = 0;
+    int k = 0;
+
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (replies[i].valid == 1) {
+            gf_msg_debug(this->name, 0,
+                         "Child:%s mtime_sec = %" PRId64 ", size = %" PRIu64
+                         " for gfid %s",
+                         priv->children[i]->name, replies[i].poststat.ia_mtime,
+                         replies[i].poststat.ia_size, uuid_utoa(inode->gfid));
+            vote_count = 0;
+            for (k = 0; k < priv->child_count; k++) {
+                if ((replies[k].poststat.ia_mtime ==
+                     replies[i].poststat.ia_mtime) &&
+                    (replies[k].poststat.ia_size ==
+                     replies[i].poststat.ia_size)) {
+                    vote_count++;
                 }
+            }
+            if (vote_count > priv->child_count / 2) {
+                fav_child = i;
+                break;
+            }
         }
+    }
+    return fav_child;
+}
 
-        if (type == AFR_SELF_HEAL_DATA) {
-                size_differs = afr_sh_mark_if_size_differs (sh, child_count);
+/*
+ * afr_sh_fav_by_mtime: Choose favorite child by mtime.
+ */
+int
+afr_sh_fav_by_mtime(xlator_t *this, struct afr_reply *replies, inode_t *inode)
+{
+    afr_private_t *priv;
+    int fav_child = -1;
+    int i = 0;
+    uint32_t cmp_mtime = 0;
+    uint32_t cmp_mtime_nsec = 0;
+
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (replies[i].valid == 1) {
+            gf_msg_debug(this->name, 0,
+                         "Child:%s mtime = %" PRId64
+                         ", mtime_nsec = %d for "
+                         "gfid %s",
+                         priv->children[i]->name, replies[i].poststat.ia_mtime,
+                         replies[i].poststat.ia_mtime_nsec,
+                         uuid_utoa(inode->gfid));
+            if (replies[i].poststat.ia_mtime > cmp_mtime) {
+                cmp_mtime = replies[i].poststat.ia_mtime;
+                cmp_mtime_nsec = replies[i].poststat.ia_mtime_nsec;
+                fav_child = i;
+            } else if ((replies[i].poststat.ia_mtime == cmp_mtime) &&
+                       (replies[i].poststat.ia_mtime_nsec > cmp_mtime_nsec)) {
+                cmp_mtime = replies[i].poststat.ia_mtime;
+                cmp_mtime_nsec = replies[i].poststat.ia_mtime_nsec;
+                fav_child = i;
+            }
         }
+    }
+    return fav_child;
+}
 
-        if ((type == AFR_SELF_HEAL_METADATA)
-            && afr_sh_all_nodes_innocent (characters, child_count)) {
+/*
+ * afr_sh_fav_by_ctime: Choose favorite child by ctime.
+ */
+int
+afr_sh_fav_by_ctime(xlator_t *this, struct afr_reply *replies, inode_t *inode)
+{
+    afr_private_t *priv;
+    int fav_child = -1;
+    int i = 0;
+    uint32_t cmp_ctime = 0;
+    uint32_t cmp_ctime_nsec = 0;
+
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (replies[i].valid == 1) {
+            gf_msg_debug(this->name, 0,
+                         "Child:%s ctime = %" PRId64
+                         ", ctime_nsec = %d for "
+                         "gfid %s",
+                         priv->children[i]->name, replies[i].poststat.ia_ctime,
+                         replies[i].poststat.ia_ctime_nsec,
+                         uuid_utoa(inode->gfid));
+            if (replies[i].poststat.ia_ctime > cmp_ctime) {
+                cmp_ctime = replies[i].poststat.ia_ctime;
+                cmp_ctime_nsec = replies[i].poststat.ia_ctime_nsec;
+                fav_child = i;
+            } else if ((replies[i].poststat.ia_ctime == cmp_ctime) &&
+                       (replies[i].poststat.ia_ctime_nsec > cmp_ctime_nsec)) {
+                cmp_ctime = replies[i].poststat.ia_ctime;
+                cmp_ctime_nsec = replies[i].poststat.ia_ctime_nsec;
+                fav_child = i;
+            }
+        }
+    }
+    return fav_child;
+}
 
-                nsources = afr_sh_mark_loweia_uid_as_source (sh, child_count);
-                goto out;
+/*
+ * afr_sh_fav_by_size: Choose favorite child by size
+ * when not all files are of zero size.
+ */
+int
+afr_sh_fav_by_size(xlator_t *this, struct afr_reply *replies, inode_t *inode)
+{
+    afr_private_t *priv;
+    int fav_child = -1;
+    int i = 0;
+    uint64_t cmp_sz = 0;
+
+    priv = this->private;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid) {
+            continue;
         }
+        gf_msg_debug(this->name, 0,
+                     "Child:%s file size = %" PRIu64 " for gfid %s",
+                     priv->children[i]->name, replies[i].poststat.ia_size,
+                     uuid_utoa(inode->gfid));
+        if (replies[i].poststat.ia_type == IA_IFDIR) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SBRAIN_FAV_CHILD_POLICY,
+                   "Cannot perform selfheal on %s. "
+                   "Size policy is not applicable to directories.",
+                   uuid_utoa(inode->gfid));
+            break;
+        }
+        if (replies[i].poststat.ia_size > cmp_sz) {
+            cmp_sz = replies[i].poststat.ia_size;
+            fav_child = i;
+        } else if (replies[i].poststat.ia_size == cmp_sz) {
+            fav_child = -1;
+        }
+    }
+    if (fav_child == -1) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
+               "No bigger file");
+    }
+    return fav_child;
+}
 
-        if (afr_sh_all_nodes_innocent (characters, child_count)) {
-                if (size_differs) {
-                        nsources = afr_sh_mark_biggest_as_source (sh,
-                                                                  child_count);
-                }
+int
+afr_sh_get_fav_by_policy(xlator_t *this, struct afr_reply *replies,
+                         inode_t *inode, char **policy_str)
+{
+    afr_private_t *priv = NULL;
+    int fav_child = -1;
+
+    priv = this->private;
+    if (!afr_can_decide_split_brain_source_sinks(replies, priv->child_count)) {
+        return -1;
+    }
+
+    switch (priv->fav_child_policy) {
+        case AFR_FAV_CHILD_BY_SIZE:
+            fav_child = afr_sh_fav_by_size(this, replies, inode);
+            if (policy_str && fav_child >= 0) {
+                *policy_str = "SIZE";
+            }
+            break;
+        case AFR_FAV_CHILD_BY_CTIME:
+            fav_child = afr_sh_fav_by_ctime(this, replies, inode);
+            if (policy_str && fav_child >= 0) {
+                *policy_str = "CTIME";
+            }
+            break;
+        case AFR_FAV_CHILD_BY_MTIME:
+            fav_child = afr_sh_fav_by_mtime(this, replies, inode);
+            if (policy_str && fav_child >= 0) {
+                *policy_str = "MTIME";
+            }
+            break;
+        case AFR_FAV_CHILD_BY_MAJORITY:
+            fav_child = afr_sh_fav_by_majority(this, replies, inode);
+            if (policy_str && fav_child >= 0) {
+                *policy_str = "MAJORITY";
+            }
+            break;
+        case AFR_FAV_CHILD_NONE:
+        default:
+            break;
+    }
+
+    return fav_child;
+}
 
-        } else if (afr_sh_wise_nodes_exist (characters, child_count)) {
-                afr_sh_compute_wisdom (pending_matrix, characters, child_count);
+int
+afr_mark_split_brain_source_sinks_by_policy(
+    call_frame_t *frame, xlator_t *this, inode_t *inode, unsigned char *sources,
+    unsigned char *sinks, unsigned char *healed_sinks, unsigned char *locked_on,
+    struct afr_reply *replies, afr_transaction_type type)
+{
+    afr_private_t *priv = NULL;
+    int fav_child = -1;
+    char mtime_str[256];
+    char ctime_str[256];
+    char *policy_str = NULL;
+    struct tm *tm_ptr;
+    time_t time;
+
+    priv = this->private;
+
+    fav_child = afr_sh_get_fav_by_policy(this, replies, inode, &policy_str);
+    if (fav_child == -1) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SBRAIN_FAV_CHILD_POLICY,
+               "No child selected by favorite-child policy.");
+    } else if (fav_child > priv->child_count - 1) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SBRAIN_FAV_CHILD_POLICY,
+               "Invalid child (%d) "
+               "selected by policy %s.",
+               fav_child, policy_str);
+    } else if (fav_child >= 0) {
+        time = replies[fav_child].poststat.ia_mtime;
+        tm_ptr = localtime(&time);
+        strftime(mtime_str, sizeof(mtime_str), "%Y-%m-%d %H:%M:%S", tm_ptr);
+        time = replies[fav_child].poststat.ia_ctime;
+        tm_ptr = localtime(&time);
+        strftime(ctime_str, sizeof(ctime_str), "%Y-%m-%d %H:%M:%S", tm_ptr);
+
+        gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_SBRAIN_FAV_CHILD_POLICY,
+               "Source %s selected as authentic to resolve conflicting data "
+               "in file (gfid:%s) by %s (%" PRIu64
+               " bytes @ %s mtime, %s "
+               "ctime).",
+               priv->children[fav_child]->name, uuid_utoa(inode->gfid),
+               policy_str, replies[fav_child].poststat.ia_size, mtime_str,
+               ctime_str);
+
+        sources[fav_child] = 1;
+        sinks[fav_child] = 0;
+        healed_sinks[fav_child] = 0;
+    }
+    return fav_child;
+}
 
-                if (afr_sh_wise_nodes_conflict (characters, child_count)) {
-                        /* split-brain */
+gf_boolean_t
+afr_is_file_empty_on_all_children(afr_private_t *priv,
+                                  struct afr_reply *replies)
+{
+    int i = 0;
 
-                        nsources = -1;
-                        goto out;
+    for (i = 0; i < priv->child_count; i++) {
+        if ((!replies[i].valid) || (replies[i].op_ret != 0) ||
+            (replies[i].poststat.ia_size != 0))
+            return _gf_false;
+    }
 
-                } else {
-                        nsources = afr_sh_mark_wisest_as_sources (sources, 
-                                                                  characters,
-                                                                  child_count);
-                }
-        } else {
-                nsources = afr_sh_mark_biggest_fool_as_source (sh, characters,
-                                                               child_count);
-        }
+    return _gf_true;
+}
 
-out:
-        GF_FREE (characters);
+int
+afr_mark_source_sinks_if_file_empty(xlator_t *this, unsigned char *sources,
+                                    unsigned char *sinks,
+                                    unsigned char *healed_sinks,
+                                    unsigned char *locked_on,
+                                    struct afr_reply *replies,
+                                    afr_transaction_type type)
+{
+    int source = -1;
+    int i = 0;
+    afr_private_t *priv = this->private;
+    struct iatt stbuf = {
+        0,
+    };
+
+    if ((AFR_COUNT(locked_on, priv->child_count) < priv->child_count) ||
+        (afr_success_count(replies, priv->child_count) < priv->child_count))
+        return -1;
+
+    if (type == AFR_DATA_TRANSACTION) {
+        if (!afr_is_file_empty_on_all_children(priv, replies))
+            return -1;
+        goto mark;
+    }
+
+    /*For AFR_METADATA_TRANSACTION, metadata must be same on all bricks.*/
+    stbuf = replies[0].poststat;
+    for (i = 1; i < priv->child_count; i++) {
+        if ((!IA_EQUAL(stbuf, replies[i].poststat, type)) ||
+            (!IA_EQUAL(stbuf, replies[i].poststat, uid)) ||
+            (!IA_EQUAL(stbuf, replies[i].poststat, gid)) ||
+            (!IA_EQUAL(stbuf, replies[i].poststat, prot)))
+            return -1;
+    }
+    for (i = 1; i < priv->child_count; i++) {
+        if (!afr_xattrs_are_equal(replies[0].xdata, replies[i].xdata))
+            return -1;
+    }
+
+mark:
+    /* data/metadata is same on all bricks. Pick one of them as source. Rest
+     * are sinks.*/
+    for (i = 0; i < priv->child_count; i++) {
+        if (source == -1) {
+            source = i;
+            sources[i] = 1;
+            sinks[i] = 0;
+            healed_sinks[i] = 0;
+            continue;
+        }
+        sources[i] = 0;
+        sinks[i] = 1;
+        healed_sinks[i] = 1;
+    }
 
-	return nsources;
+    return source;
 }
 
+/* Return a source depending on the type of heal_op, and set sources[source],
+ * sinks[source] and healed_sinks[source] to 1, 0 and 0 respectively. Do so
+ * only if the following condition is met:
+ * ∀i((i ∈ locked_on[] ∧ i=1)==>(sources[i]=0 ∧ sinks[i]=1 ∧ healed_sinks[i]=1))
+ * i.e. for each locked node, sources[node] is 0; healed_sinks[node] and
+ * sinks[node] are 1. This should be the case if the file is in split-brain.
+ */
+int
+afr_mark_split_brain_source_sinks(
+    call_frame_t *frame, xlator_t *this, inode_t *inode, unsigned char *sources,
+    unsigned char *sinks, unsigned char *healed_sinks, unsigned char *locked_on,
+    struct afr_reply *replies, afr_transaction_type type)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    dict_t *xdata_req = NULL;
+    int heal_op = -1;
+    int ret = -1;
+    int source = -1;
+
+    local = frame->local;
+    priv = this->private;
+    xdata_req = local->xdata_req;
+
+    source = afr_mark_source_sinks_if_file_empty(
+        this, sources, sinks, healed_sinks, locked_on, replies, type);
+    if (source >= 0)
+        return source;
+
+    ret = dict_get_int32_sizen(xdata_req, "heal-op", &heal_op);
+    if (ret)
+        goto autoheal;
+
+    source = afr_mark_split_brain_source_sinks_by_heal_op(
+        frame, this, sources, sinks, healed_sinks, locked_on, replies, type,
+        heal_op);
+    return source;
+
+autoheal:
+    /* Automatically heal if fav_child_policy is set. */
+    if (priv->fav_child_policy != AFR_FAV_CHILD_NONE) {
+        source = afr_mark_split_brain_source_sinks_by_policy(
+            frame, this, inode, sources, sinks, healed_sinks, locked_on,
+            replies, type);
+        if (source != -1) {
+            ret = dict_set_int32_sizen(xdata_req, "fav-child-policy", 1);
+            if (ret)
+                return -1;
+        }
+    }
 
-void
-afr_sh_pending_to_delta (afr_private_t *priv, dict_t **xattr,
-                         int32_t *delta_matrix[], int success[],
-                         int child_count, afr_transaction_type type)
-{
-	int i = 0;
-	int j = 0;
-        int k = 0;
-
-	/* Indexable by result of afr_index_for_transaction_type(): 0 -- 2. */
-	int32_t   pending[3];
-	void    * pending_raw = NULL;
-        int       ret         = 0;
-
-	/* start clean */
-	for (i = 0; i < child_count; i++) {
-		for (j = 0; j < child_count; j++) {
-			delta_matrix[i][j] = 0;
-		}
-	}
-
-	for (i = 0; i < child_count; i++) {
-                pending_raw = NULL;
-
-                for (j = 0; j < child_count; j++) {
-                        ret = dict_get_ptr (xattr[i], priv->pending_key[j],
-                                            &pending_raw);
-                        if (ret < 0)
-                                gf_log ("afr_sh_pending_to_delta",
-                                        GF_LOG_DEBUG,
-                                        "Unable to get dict value.");
-
-                        if (!success[j])
-                                continue;
-
-                        k = afr_index_for_transaction_type (type);
-
-                        if (pending_raw) {
-				memcpy (pending, pending_raw, sizeof(pending));
-                                delta_matrix[i][j] = -(ntoh32 (pending[k]));
-                        } else {
-                                delta_matrix[i][j]  = 0;
-                        }
-
-                }
-	}
+    return source;
 }
 
-
 int
-afr_sh_delta_to_xattr (afr_private_t *priv,
-                       int32_t *delta_matrix[], dict_t *xattr[],
-		       int child_count, afr_transaction_type type)
+_afr_fav_child_reset_sink_xattrs(call_frame_t *frame, xlator_t *this,
+                                 inode_t *inode, int source,
+                                 unsigned char *healed_sinks,
+                                 unsigned char *undid_pending,
+                                 afr_transaction_type type,
+                                 unsigned char *locked_on,
+                                 struct afr_reply *replies)
 {
-	int i = 0;
-	int j = 0;
-        int k = 0;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int *input_dirty = NULL;
+    int **input_matrix = NULL;
+    int *output_dirty = NULL;
+    int **output_matrix = NULL;
+    dict_t *xattr = NULL;
+    dict_t *xdata = NULL;
+    int i = 0;
+
+    priv = this->private;
+    local = frame->local;
+
+    if (!dict_get_sizen(local->xdata_req, "fav-child-policy"))
+        return 0;
 
-	int ret = 0;
+    xdata = dict_new();
+    if (!xdata)
+        return -1;
 
-	int32_t *pending = 0;
+    input_dirty = alloca0(priv->child_count * sizeof(int));
+    input_matrix = ALLOC_MATRIX(priv->child_count, int);
+    output_dirty = alloca0(priv->child_count * sizeof(int));
+    output_matrix = ALLOC_MATRIX(priv->child_count, int);
 
-	for (i = 0; i < child_count; i++) {
-		if (!xattr[i])
-			continue;
+    afr_selfheal_extract_xattr(this, replies, type, input_dirty, input_matrix);
 
-		for (j = 0; j < child_count; j++) {
-                        pending = GF_CALLOC (sizeof (int32_t), 3,
-                                             gf_afr_mt_int32_t);
-                        /* 3 = data+metadata+entry */
+    for (i = 0; i < priv->child_count; i++) {
+        if (i == source || !healed_sinks[i])
+            continue;
+        output_dirty[i] = -input_dirty[i];
+        output_matrix[i][source] = -input_matrix[i][source];
+    }
 
-                        k = afr_index_for_transaction_type (type);
+    for (i = 0; i < priv->child_count; i++) {
+        if (!healed_sinks[i] || !locked_on[i])
+            continue;
+        xattr = afr_selfheal_output_xattr(this, _gf_false, type, output_dirty,
+                                          output_matrix, i, NULL);
 
-			pending[k] = hton32 (delta_matrix[i][j]);
+        afr_selfheal_post_op(frame, this, inode, i, xattr, xdata);
 
-                        ret = dict_set_bin (xattr[i], priv->pending_key[j],
-                                            pending,
-                                            3 * sizeof (int32_t));
-                        if (ret < 0)
-                                gf_log ("afr_sh_delta_to_xattr",
-                                        GF_LOG_WARNING,
-                                        "Unable to set dict value.");
-		}
-	}
-	return 0;
-}
+        undid_pending[i] = 1;
+        dict_unref(xattr);
+    }
 
+    if (xdata)
+        dict_unref(xdata);
 
-int
-afr_sh_has_metadata_pending (dict_t *xattr, int child_count, xlator_t *this)
+    return 0;
+}
+
+gf_boolean_t
+afr_does_witness_exist(xlator_t *this, uint64_t *witness)
 {
-	afr_private_t *priv = NULL;
-	/* Indexable by result of afr_index_for_transaction_type(): 0 -- 2. */
-	int32_t       pending[3];
-	void          *pending_raw = NULL;
+    int i = 0;
+    afr_private_t *priv = NULL;
 
-	int           ret = -1;
-	int            i  = 0;
-        int            j  = 0;
+    priv = this->private;
 
-	priv = this->private;
+    for (i = 0; i < priv->child_count; i++) {
+        if (witness[i])
+            return _gf_true;
+    }
+    return _gf_false;
+}
 
-        for (i = 0; i < priv->child_count; i++) {
-                ret = dict_get_ptr (xattr, priv->pending_key[i],
-                                    &pending_raw);
+unsigned int
+afr_get_quorum_count(afr_private_t *priv)
+{
+    if (priv->quorum_count == AFR_QUORUM_AUTO) {
+        return priv->child_count / 2 + 1;
+    } else {
+        return priv->quorum_count;
+    }
+}
 
-                if (ret != 0)
-                        return 0;
+void
+afr_selfheal_post_op_failure_accounting(afr_private_t *priv, char *accused,
+                                        unsigned char *sources,
+                                        unsigned char *locked_on)
+{
+    int i = 0;
+    unsigned int quorum_count = 0;
 
-		memcpy (pending, pending_raw, sizeof(pending));
-                j = afr_index_for_transaction_type (AFR_METADATA_TRANSACTION);
+    if (AFR_COUNT(sources, priv->child_count) != 0)
+        return;
 
-                if (pending[j])
-                        return 1;
+    quorum_count = afr_get_quorum_count(priv);
+    for (i = 0; i < priv->child_count; i++) {
+        if ((accused[i] < quorum_count) && locked_on[i]) {
+            sources[i] = 1;
         }
-
-	return 0;
+    }
+    return;
 }
 
+/*
+ * This function determines if a self-heal is required for a given inode,
+ * and if needed, in what direction.
+ *
+ * locked_on[] is the array representing servers which have been locked and
+ * from which xattrs have been fetched for analysis.
+ *
+ * The output of the function is by filling the arrays sources[] and sinks[].
+ *
+ * sources[i] is set if i'th server is an eligible source for a selfheal.
+ *
+ * sinks[i] is set if i'th server needs to be healed.
+ *
+ * if sources[0..N] are all set, there is no need for a selfheal.
+ *
+ * if sinks[0..N] are all set, the inode is in split brain.
+ *
+ */
 
 int
-afr_sh_has_data_pending (dict_t *xattr, int child_count, xlator_t *this)
+afr_selfheal_find_direction(call_frame_t *frame, xlator_t *this,
+                            struct afr_reply *replies,
+                            afr_transaction_type type, unsigned char *locked_on,
+                            unsigned char *sources, unsigned char *sinks,
+                            uint64_t *witness, unsigned char *pflag)
 {
-	afr_private_t *priv = NULL;
-	/* Indexable by result of afr_index_for_transaction_type(): 0 -- 2. */
-	int32_t       pending[3];
-	void          *pending_raw = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    int j = 0;
+    int *dirty = NULL;         /* Denotes if dirty xattr is set */
+    int **matrix = NULL;       /* Changelog matrix */
+    char *accused = NULL;      /* Accused others without any self-accusal */
+    char *pending = NULL;      /* Have pending operations on others */
+    char *self_accused = NULL; /* Accused itself */
+
+    priv = this->private;
+
+    dirty = alloca0(priv->child_count * sizeof(int));
+    accused = alloca0(priv->child_count);
+    pending = alloca0(priv->child_count);
+    self_accused = alloca0(priv->child_count);
+    matrix = ALLOC_MATRIX(priv->child_count, int);
+    memset(witness, 0, sizeof(*witness) * priv->child_count);
+
+    /* First construct the pending matrix for further analysis */
+    afr_selfheal_extract_xattr(this, replies, type, dirty, matrix);
+
+    if (pflag) {
+        for (i = 0; i < priv->child_count; i++) {
+            for (j = 0; j < priv->child_count; j++)
+                if (matrix[i][j])
+                    *pflag |= PFLAG_PENDING;
+            if (*pflag)
+                break;
+        }
+    }
+
+    if (afr_success_count(replies, priv->child_count) < priv->child_count) {
+        /* Treat this just like locks not being acquired */
+        return -ENOTCONN;
+    }
+
+    /* short list all self-accused */
+    for (i = 0; i < priv->child_count; i++) {
+        if (matrix[i][i])
+            self_accused[i] = 1;
+    }
+
+    /* Next short list all accused to exclude them from being sources */
+    /* Self-accused can't accuse others as they are FOOLs */
+    for (i = 0; i < priv->child_count; i++) {
+        for (j = 0; j < priv->child_count; j++) {
+            if (matrix[i][j]) {
+                if (!self_accused[i])
+                    accused[j] += 1;
+                if (i != j)
+                    pending[i] += 1;
+            }
+        }
+    }
 
-	int           ret = -1;
-	int            i  = 0;
-        int            j  = 0;
+    /* Short list all non-accused as sources */
+    for (i = 0; i < priv->child_count; i++) {
+        if (!accused[i] && locked_on[i])
+            sources[i] = 1;
+        else
+            sources[i] = 0;
+    }
+
+    /* Everyone accused by non-self-accused sources are sinks */
+    memset(sinks, 0, priv->child_count);
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i])
+            continue;
+        if (self_accused[i])
+            continue;
+        for (j = 0; j < priv->child_count; j++) {
+            if (matrix[i][j])
+                sinks[j] = 1;
+        }
+    }
+
+    /* For breaking ties provide with number of fops they witnessed */
+
+    /*
+     * count the pending fops witnessed from itself to others when it is
+     * self-accused
+     */
+    for (i = 0; i < priv->child_count; i++) {
+        if (!self_accused[i])
+            continue;
+        for (j = 0; j < priv->child_count; j++) {
+            if (i == j)
+                continue;
+            witness[i] += matrix[i][j];
+        }
+    }
 
-	priv = this->private;
+    if (type == AFR_DATA_TRANSACTION || type == AFR_METADATA_TRANSACTION)
+        afr_selfheal_post_op_failure_accounting(priv, accused, sources,
+                                                locked_on);
 
+    /* If no sources, all locked nodes are sinks - split brain */
+    if (AFR_COUNT(sources, priv->child_count) == 0) {
         for (i = 0; i < priv->child_count; i++) {
-                ret = dict_get_ptr (xattr, priv->pending_key[i],
-                                    &pending_raw);
-
-                if (ret != 0)
-                        return 0;
-                
-		memcpy (pending, pending_raw, sizeof(pending));
-                j = afr_index_for_transaction_type (AFR_DATA_TRANSACTION);
-
-                if (pending[j])
-                        return 1;
+            if (locked_on[i])
+                sinks[i] = 1;
         }
+        if (pflag)
+            *pflag |= PFLAG_SBRAIN;
+    }
+
+    /* One more class of witness similar to dirty in v2 is where no pending
+     * exists but we have self-accusing markers. This can happen in afr-v1
+     * if the brick crashes just after doing xattrop on self but
+     * before xattrop on the other xattrs on the brick in pre-op. */
+    if (AFR_COUNT(pending, priv->child_count) == 0) {
+        for (i = 0; i < priv->child_count; i++) {
+            if (self_accused[i])
+                witness[i] += matrix[i][i];
+        }
+    } else {
+        /* In afr-v1 if a file is self-accused and has pending
+         * operations on others then it is similar to 'dirty' in afr-v2.
+         * Consider such cases as witness.
+         */
+        for (i = 0; i < priv->child_count; i++) {
+            if (self_accused[i] && pending[i])
+                witness[i] += matrix[i][i];
+        }
+    }
+
+    /* count the number of dirty fops witnessed */
+    for (i = 0; i < priv->child_count; i++)
+        witness[i] += dirty[i];
 
-	return 0;
+    return 0;
 }
 
+void
+afr_log_selfheal(uuid_t gfid, xlator_t *this, int ret, char *type, int source,
+                 unsigned char *sources, unsigned char *healed_sinks)
+{
+    char *status = NULL;
+    char *sinks_str = NULL;
+    char *p = NULL;
+    char *sources_str = NULL;
+    char *q = NULL;
+    afr_private_t *priv = NULL;
+    gf_loglevel_t loglevel = GF_LOG_NONE;
+    int i = 0;
+
+    priv = this->private;
+    sinks_str = alloca0(priv->child_count * 8);
+    p = sinks_str;
+    sources_str = alloca0(priv->child_count * 8);
+    q = sources_str;
+    for (i = 0; i < priv->child_count; i++) {
+        if (healed_sinks[i])
+            p += sprintf(p, "%d ", i);
+        if (sources[i]) {
+            if (source == i) {
+                q += sprintf(q, "[%d] ", i);
+            } else {
+                q += sprintf(q, "%d ", i);
+            }
+        }
+    }
+
+    if (ret < 0) {
+        status = "Failed";
+        loglevel = GF_LOG_DEBUG;
+    } else {
+        status = "Completed";
+        loglevel = GF_LOG_INFO;
+    }
+
+    gf_msg(this->name, loglevel, 0, AFR_MSG_SELF_HEAL_INFO,
+           "%s %s selfheal on %s. "
+           "sources=%s sinks=%s",
+           status, type, uuid_utoa(gfid), sources_str, sinks_str);
+}
 
 int
-afr_sh_has_entry_pending (dict_t *xattr, int child_count, xlator_t *this)
+afr_selfheal_discover_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int op_ret, int op_errno, inode_t *inode,
+                          struct iatt *buf, dict_t *xdata, struct iatt *parbuf)
 {
-        afr_private_t *priv = NULL;
-	/* Indexable by result of afr_index_for_transaction_type(): 0 -- 2. */
-	int32_t       pending[3];
-	void          *pending_raw = NULL;
+    afr_local_t *local = NULL;
+    int i = -1;
+    GF_UNUSED int ret = -1;
+    int8_t need_heal = 1;
+
+    local = frame->local;
+    i = (long)cookie;
+
+    local->replies[i].valid = 1;
+    local->replies[i].op_ret = op_ret;
+    local->replies[i].op_errno = op_errno;
+    if (buf)
+        local->replies[i].poststat = *buf;
+    if (parbuf)
+        local->replies[i].postparent = *parbuf;
+    if (xdata) {
+        local->replies[i].xdata = dict_ref(xdata);
+        ret = dict_get_int8(xdata, "link-count", &need_heal);
+    }
+
+    local->replies[i].need_heal = need_heal;
+    syncbarrier_wake(&local->barrier);
+
+    return 0;
+}
 
-	int           ret = -1;
-	int            i  = 0;
-        int            j  = 0;
+inode_t *
+afr_selfheal_unlocked_lookup_on(call_frame_t *frame, inode_t *parent,
+                                const char *name, struct afr_reply *replies,
+                                unsigned char *lookup_on, dict_t *xattr)
+{
+    loc_t loc = {
+        0,
+    };
+    dict_t *xattr_req = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    inode_t *inode = NULL;
 
-	priv = this->private;
+    local = frame->local;
+    priv = frame->this->private;
 
-        for (i = 0; i < priv->child_count; i++) {
-                ret = dict_get_ptr (xattr, priv->pending_key[i],
-                                    &pending_raw);
+    xattr_req = dict_new();
+    if (!xattr_req)
+        return NULL;
 
-                if (ret != 0)
-                        return 0;
-                
-		memcpy (pending, pending_raw, sizeof(pending));
-                j = afr_index_for_transaction_type (AFR_ENTRY_TRANSACTION);
+    if (xattr)
+        dict_copy(xattr, xattr_req);
 
-                if (pending[j])
-                        return 1;
-        }
+    if (afr_xattr_req_prepare(frame->this, xattr_req) != 0) {
+        dict_unref(xattr_req);
+        return NULL;
+    }
 
-	return 0;
-}
+    inode = inode_new(parent->table);
+    if (!inode) {
+        dict_unref(xattr_req);
+        return NULL;
+    }
 
+    loc.parent = inode_ref(parent);
+    gf_uuid_copy(loc.pargfid, parent->gfid);
+    loc.name = name;
+    loc.inode = inode_ref(inode);
 
-/**
- * is_matrix_zero - return true if pending matrix is all zeroes
- */
+    AFR_ONLIST(lookup_on, frame, afr_selfheal_discover_cbk, lookup, &loc,
+               xattr_req);
 
-int
-afr_sh_is_matrix_zero (int32_t *pending_matrix[], int child_count)
-{
-	int i, j;
+    afr_replies_copy(replies, local->replies, priv->child_count);
+
+    loc_wipe(&loc);
+    dict_unref(xattr_req);
 
-	for (i = 0; i < child_count; i++) 
-		for (j = 0; j < child_count; j++) 
-			if (pending_matrix[i][j]) 
-				return 0;
-	return 1;
+    return inode;
 }
 
+static int
+afr_set_multi_dom_lock_count_request(xlator_t *this, dict_t *dict)
+{
+    int ret = 0;
+    afr_private_t *priv = NULL;
+    char *key1 = NULL;
+    char *key2 = NULL;
+
+    priv = this->private;
+    key1 = alloca0(strlen(GLUSTERFS_INODELK_DOM_PREFIX) + 2 +
+                   strlen(this->name));
+    key2 = alloca0(strlen(GLUSTERFS_INODELK_DOM_PREFIX) + 2 +
+                   strlen(priv->sh_domain));
+
+    ret = dict_set_uint32(dict, GLUSTERFS_MULTIPLE_DOM_LK_CNT_REQUESTS, 1);
+    if (ret)
+        return ret;
+
+    sprintf(key1, "%s:%s", GLUSTERFS_INODELK_DOM_PREFIX, this->name);
+    ret = dict_set_uint32(dict, key1, 1);
+    if (ret)
+        return ret;
+
+    sprintf(key2, "%s:%s", GLUSTERFS_INODELK_DOM_PREFIX, priv->sh_domain);
+    ret = dict_set_uint32(dict, key2, 1);
+    if (ret)
+        return ret;
+
+    return 0;
+}
 
 int
-afr_sh_missing_entries_done (call_frame_t *frame, xlator_t *this)
+afr_selfheal_unlocked_discover_on(call_frame_t *frame, inode_t *inode,
+                                  uuid_t gfid, struct afr_reply *replies,
+                                  unsigned char *discover_on, dict_t *dict)
 {
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-	int              i = 0;
+    loc_t loc = {
+        0,
+    };
+    dict_t *xattr_req = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
+    local = frame->local;
+    priv = frame->this->private;
 
-//	memset (sh->child_errno, 0, sizeof (int) * priv->child_count);
-	memset (sh->buf, 0, sizeof (struct iatt) * priv->child_count);
+    xattr_req = dict_new();
+    if (!xattr_req)
+        return -ENOMEM;
+    if (dict)
+        dict_copy(dict, xattr_req);
 
-        for (i = 0; i < priv->child_count; i++) {
-                sh->locked_nodes[i] = 0;
-        }
+    if (afr_xattr_req_prepare(frame->this, xattr_req) != 0) {
+        dict_unref(xattr_req);
+        return -ENOMEM;
+    }
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (sh->xattr[i])
-			dict_unref (sh->xattr[i]);
-		sh->xattr[i] = NULL;
-	}
+    if (afr_set_multi_dom_lock_count_request(frame->this, xattr_req)) {
+        dict_unref(xattr_req);
+        return -1;
+    }
 
-	if (local->govinda_gOvinda) {
-		gf_log (this->name, GF_LOG_TRACE,
-			"aborting selfheal of %s",
-			local->loc.path);
-		sh->completion_cbk (frame, this);
-	} else {
-		gf_log (this->name, GF_LOG_TRACE,
-			"proceeding to metadata check on %s",
-			local->loc.path);
-		afr_self_heal_metadata (frame, this);
-	}
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, gfid);
 
-	return 0;
-}
+    AFR_ONLIST(discover_on, frame, afr_selfheal_discover_cbk, lookup, &loc,
+               xattr_req);
 
+    afr_replies_copy(replies, local->replies, priv->child_count);
 
-static int
-sh_missing_entries_finish (call_frame_t *frame, xlator_t *this)
+    loc_wipe(&loc);
+    dict_unref(xattr_req);
+
+    return 0;
+}
+
+int
+afr_selfheal_unlocked_discover(call_frame_t *frame, inode_t *inode, uuid_t gfid,
+                               struct afr_reply *replies)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
+    afr_local_t *local = NULL;
+    dict_t *dict = NULL;
 
-	local = frame->local;
-        int_lock = &local->internal_lock;
+    local = frame->local;
 
-        int_lock->lock_cbk = afr_sh_missing_entries_done;
-        afr_unlock (frame, this);
+    if (local->xattr_req)
+        dict = local->xattr_req;
 
-	return 0;
+    return afr_selfheal_unlocked_discover_on(frame, inode, gfid, replies,
+                                             local->child_up, dict);
 }
 
-
-static int
-sh_destroy_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		int32_t op_ret, int op_errno,
-                struct iatt *preop, struct iatt *postop)
+unsigned int
+afr_success_count(struct afr_reply *replies, unsigned int count)
 {
-        afr_local_t *local = NULL;
+    int i = 0;
+    unsigned int success = 0;
 
-        loc_t *parent_loc = cookie;
+    for (i = 0; i < count; i++)
+        if (replies[i].valid && replies[i].op_ret == 0)
+            success++;
+    return success;
+}
 
-        int call_count = 0;
+int
+afr_selfheal_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    int i = 0;
 
-        local = frame->local;
+    local = frame->local;
+    i = (long)cookie;
 
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "setattr on %s failed: %s",
-                        local->loc.path, strerror (op_errno));
-        }
+    local->replies[i].valid = 1;
+    local->replies[i].op_ret = op_ret;
+    local->replies[i].op_errno = op_errno;
 
-        if (parent_loc) {
-                loc_wipe (parent_loc);
-                GF_FREE (parent_loc);
-        }
+    syncbarrier_wake(&local->barrier);
 
-        call_count = afr_frame_return (frame);
-        
-        if (call_count == 0) {
-                STACK_DESTROY (frame->root);
-        }
-        
-	return 0;
+    return 0;
 }
 
+int
+afr_locked_fill(call_frame_t *frame, xlator_t *this, unsigned char *locked_on)
+{
+    int i = 0;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int count = 0;
+
+    local = frame->local;
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->replies[i].valid && local->replies[i].op_ret == 0) {
+            locked_on[i] = 1;
+            count++;
+        } else {
+            locked_on[i] = 0;
+        }
+    }
 
-static int
-sh_missing_entries_newentry_cbk (call_frame_t *frame, void *cookie,
-				 xlator_t *this,
-				 int32_t op_ret, int32_t op_errno,
-				 inode_t *inode, struct iatt *buf,
-                                 struct iatt *preparent,
-                                 struct iatt *postparent)
-{
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-	call_frame_t    *setattr_frame = NULL;
-	int              call_count = 0;
-	int              child_index = 0;
-
-        loc_t *parent_loc = NULL;
-
-	struct iatt     stbuf;
-        int32_t valid;
-
-	local = frame->local;
-	sh    = &local->self_heal;
-	priv  = this->private;
-
-	child_index = (long) cookie;
-
-	stbuf.ia_atime = sh->buf[sh->source].ia_atime;
-	stbuf.ia_atime_nsec = sh->buf[sh->source].ia_atime_nsec;
-	stbuf.ia_mtime = sh->buf[sh->source].ia_mtime;
-	stbuf.ia_mtime_nsec = sh->buf[sh->source].ia_mtime_nsec;
-
-        stbuf.ia_uid = sh->buf[sh->source].ia_uid;
-        stbuf.ia_gid = sh->buf[sh->source].ia_gid;
-        
-        valid = GF_SET_ATTR_UID   | GF_SET_ATTR_GID |
-                GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME;
-        
-	if (op_ret == 0) {
-		setattr_frame = copy_frame (frame);
-                
-                setattr_frame->local = GF_CALLOC (1, sizeof (afr_local_t),
-                                                  gf_afr_mt_afr_local_t);
-
-                ((afr_local_t *)setattr_frame->local)->call_count = 2;
-
-		gf_log (this->name, GF_LOG_TRACE,
-			"setattr (%s) on subvolume %s",
-			local->loc.path, priv->children[child_index]->name);
-
-		STACK_WIND_COOKIE (setattr_frame, sh_destroy_cbk,
-                                   (void *) (long) 0,
-                                   priv->children[child_index],
-                                   priv->children[child_index]->fops->setattr,
-                                   &local->loc, &stbuf, valid);
-
-                valid      = GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME;
-                parent_loc = GF_CALLOC (1, sizeof (*parent_loc), 
-                                        gf_afr_mt_loc_t);
-                afr_build_parent_loc (parent_loc, &local->loc);
-
-                STACK_WIND_COOKIE (setattr_frame, sh_destroy_cbk,
-                                   (void *) (long) parent_loc,
-                                   priv->children[child_index],
-                                   priv->children[child_index]->fops->setattr,
-                                   parent_loc, &sh->parentbuf, valid);
-	}
-
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0) {
-		sh_missing_entries_finish (frame, this);
-	}
-
-	return 0;
+    return count;
 }
 
+int
+afr_selfheal_tryinodelk(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                        char *dom, off_t off, size_t size,
+                        unsigned char *locked_on)
+{
+    loc_t loc = {
+        0,
+    };
+    struct gf_flock flock = {
+        0,
+    };
 
-static int
-sh_missing_entries_mknod (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-	int              i = 0;
-        int              ret = 0;
-	int              enoent_count = 0;
-	int              call_count = 0;
-	mode_t           st_mode = 0;
-	dev_t            ia_dev = 0;
-        dict_t          *dict = NULL;
-
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
-
-	for (i = 0; i < priv->child_count; i++)
-		if (sh->child_errno[i] == ENOENT)
-			enoent_count++;
-
-	call_count = enoent_count;
-	local->call_count = call_count;
-
-	st_mode = st_mode_from_ia (sh->buf[sh->source].ia_prot,
-                                   sh->buf[sh->source].ia_type);
-	ia_dev  = sh->buf[sh->source].ia_dev;
-
-	gf_log (this->name, GF_LOG_TRACE,
-		"mknod %s mode 0%o on %d subvolumes",
-		local->loc.path, st_mode, enoent_count);
-
-        dict = dict_new ();
-        if (!dict)
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-
-        ret = afr_set_dict_gfid (dict, sh->buf[sh->source].ia_gfid);
-        if (ret)
-                gf_log (this->name, GF_LOG_DEBUG, "gfid set failed");
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+
+    flock.l_type = F_WRLCK;
+    flock.l_start = off;
+    flock.l_len = size;
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (sh->child_errno[i] == ENOENT) {
-			STACK_WIND_COOKIE (frame,
-					   sh_missing_entries_newentry_cbk,
-					   (void *) (long) i,
-					   priv->children[i],
-					   priv->children[i]->fops->mknod,
-					   &local->loc, st_mode, ia_dev, dict);
-			if (!--call_count)
-				break;
-		}
-	}
+    AFR_ONALL(frame, afr_selfheal_lock_cbk, inodelk, dom, &loc, F_SETLK, &flock,
+              NULL);
 
-        if (dict)
-                dict_unref (dict);
+    loc_wipe(&loc);
 
-	return 0;
+    return afr_locked_fill(frame, this, locked_on);
 }
 
+int
+afr_selfheal_inodelk(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                     char *dom, off_t off, size_t size,
+                     unsigned char *locked_on)
+{
+    loc_t loc = {
+        0,
+    };
+    struct gf_flock flock = {
+        0,
+    };
+    afr_local_t *local = NULL;
+    int i = 0;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+    local = frame->local;
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+
+    flock.l_type = F_WRLCK;
+    flock.l_start = off;
+    flock.l_len = size;
+
+    AFR_ONALL(frame, afr_selfheal_lock_cbk, inodelk, dom, &loc, F_SETLK, &flock,
+              NULL);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->replies[i].op_ret == -1 &&
+            local->replies[i].op_errno == EAGAIN) {
+            afr_locked_fill(frame, this, locked_on);
+            afr_selfheal_uninodelk(frame, this, inode, dom, off, size,
+                                   locked_on);
+
+            AFR_SEQ(frame, afr_selfheal_lock_cbk, inodelk, dom, &loc, F_SETLKW,
+                    &flock, NULL);
+            break;
+        }
+    }
 
-static int
-sh_missing_entries_mkdir (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-        dict_t          *dict = NULL;
-	int              i = 0;
-        int              ret = 0;
-	int              enoent_count = 0;
-	int              call_count = 0;
-	mode_t           st_mode = 0;
-
-
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
-
-	for (i = 0; i < priv->child_count; i++)
-		if (sh->child_errno[i] == ENOENT)
-			enoent_count++;
-
-	call_count = enoent_count;
-	local->call_count = call_count;
-
-	st_mode = st_mode_from_ia (sh->buf[sh->source].ia_prot,
-                                   sh->buf[sh->source].ia_type);
-
-        dict = dict_new ();
-        if (!dict) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory");
-                sh_missing_entries_finish (frame, this);
-                return 0;
+    loc_wipe(&loc);
+
+    return afr_locked_fill(frame, this, locked_on);
+}
+
+static void
+afr_get_lock_and_eagain_counts(afr_private_t *priv, struct afr_reply *replies,
+                               int *lock_count, int *eagain_count)
+{
+    int i = 0;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid)
+            continue;
+        if (replies[i].op_ret == 0) {
+            (*lock_count)++;
+        } else if (replies[i].op_ret == -1 && replies[i].op_errno == EAGAIN) {
+            (*eagain_count)++;
         }
+    }
+}
 
-        ret = afr_set_dict_gfid (dict, sh->buf[sh->source].ia_gfid);
-        if (ret)
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "inode gfid set failed");
+/*Do blocking locks if number of locks acquired is majority and there were some
+ * EAGAINs. Useful for odd-way replication*/
+int
+afr_selfheal_tie_breaker_inodelk(call_frame_t *frame, xlator_t *this,
+                                 inode_t *inode, char *dom, off_t off,
+                                 size_t size, unsigned char *locked_on)
+{
+    loc_t loc = {
+        0,
+    };
+    struct gf_flock flock = {
+        0,
+    };
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int lock_count = 0;
+    int eagain_count = 0;
 
+    priv = this->private;
+    local = frame->local;
 
-	gf_log (this->name, GF_LOG_TRACE,
-		"mkdir %s mode 0%o on %d subvolumes",
-		local->loc.path, st_mode, enoent_count);
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (sh->child_errno[i] == ENOENT) {
-                        if (!strcmp (local->loc.path, "/")) {
-                                /* We shouldn't try to create "/" */
+    flock.l_type = F_WRLCK;
+    flock.l_start = off;
+    flock.l_len = size;
 
-                                sh_missing_entries_finish (frame, this);
+    AFR_ONALL(frame, afr_selfheal_lock_cbk, inodelk, dom, &loc, F_SETLK, &flock,
+              NULL);
 
-                                return 0;
-                        } else {
-                                STACK_WIND_COOKIE (frame,
-                                                   sh_missing_entries_newentry_cbk,
-                                                   (void *) (long) i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->mkdir,
-                                                   &local->loc, st_mode, dict);
-                                if (!--call_count)
-                                        break;
-                        }
-		}
-	}
+    afr_get_lock_and_eagain_counts(priv, local->replies, &lock_count,
+                                   &eagain_count);
 
-        if (dict)
-                dict_unref (dict);
+    if (lock_count > priv->child_count / 2 && eagain_count) {
+        afr_locked_fill(frame, this, locked_on);
+        afr_selfheal_uninodelk(frame, this, inode, dom, off, size, locked_on);
 
-	return 0;
+        AFR_SEQ(frame, afr_selfheal_lock_cbk, inodelk, dom, &loc, F_SETLKW,
+                &flock, NULL);
+    }
+
+    loc_wipe(&loc);
+
+    return afr_locked_fill(frame, this, locked_on);
 }
 
+int
+afr_selfheal_uninodelk(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                       char *dom, off_t off, size_t size,
+                       const unsigned char *locked_on)
+{
+    loc_t loc = {
+        0,
+    };
+    struct gf_flock flock = {
+        0,
+    };
 
-static int
-sh_missing_entries_symlink (call_frame_t *frame, xlator_t *this,
-			    const char *link, struct iatt *buf)
-{
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-        dict_t          *dict = NULL;
-	int              i = 0;
-        int              ret = 0;
-	int              enoent_count = 0;
-	int              call_count = 0;
-
-
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
-
-	for (i = 0; i < priv->child_count; i++)
-		if (sh->child_errno[i] == ENOENT)
-			enoent_count++;
-
-	call_count = enoent_count;
-	local->call_count = call_count;
-
-        dict = dict_new ();
-        if (!dict) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory");
-                sh_missing_entries_finish (frame, this);
-                return 0;
-        }
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
 
-        ret = afr_set_dict_gfid (dict, buf->ia_gfid);
-        if (ret)
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "dict gfid set failed");
+    flock.l_type = F_UNLCK;
+    flock.l_start = off;
+    flock.l_len = size;
 
-	gf_log (this->name, GF_LOG_TRACE,
-		"symlink %s -> %s on %d subvolumes",
-		local->loc.path, link, enoent_count);
+    AFR_ONLIST(locked_on, frame, afr_selfheal_lock_cbk, inodelk, dom, &loc,
+               F_SETLK, &flock, NULL);
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (sh->child_errno[i] == ENOENT) {
-			STACK_WIND_COOKIE (frame,
-					   sh_missing_entries_newentry_cbk,
-					   (void *) (long) i,
-					   priv->children[i],
-					   priv->children[i]->fops->symlink,
-					   link, &local->loc, dict);
-			if (!--call_count)
-				break;
-		}
-	}
+    loc_wipe(&loc);
 
-	return 0;
+    return 0;
 }
 
-
-static int
-sh_missing_entries_readlink_cbk (call_frame_t *frame, void *cookie,
-				 xlator_t *this,
-				 int32_t op_ret, int32_t op_errno,
-				 const char *link, struct iatt *sbuf)
+int
+afr_selfheal_tryentrylk(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                        char *dom, const char *name, unsigned char *locked_on)
 {
-	if (op_ret > 0)
-		sh_missing_entries_symlink (frame, this, link, sbuf);
-	else
-		sh_missing_entries_finish (frame, this);
+    loc_t loc = {
+        0,
+    };
 
-	return 0;
-}
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
 
+    AFR_ONALL(frame, afr_selfheal_lock_cbk, entrylk, dom, &loc, name,
+              ENTRYLK_LOCK_NB, ENTRYLK_WRLCK, NULL);
 
-static int
-sh_missing_entries_readlink (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
+    loc_wipe(&loc);
 
+    return afr_locked_fill(frame, this, locked_on);
+}
 
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
+int
+afr_selfheal_entrylk(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                     char *dom, const char *name, unsigned char *locked_on)
+{
+    loc_t loc = {
+        0,
+    };
+    afr_local_t *local = NULL;
+    int i = 0;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+    local = frame->local;
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+
+    AFR_ONALL(frame, afr_selfheal_lock_cbk, entrylk, dom, &loc, name,
+              ENTRYLK_LOCK_NB, ENTRYLK_WRLCK, NULL);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->replies[i].op_ret == -1 &&
+            local->replies[i].op_errno == EAGAIN) {
+            afr_locked_fill(frame, this, locked_on);
+            afr_selfheal_unentrylk(frame, this, inode, dom, name, locked_on,
+                                   NULL);
+
+            AFR_SEQ(frame, afr_selfheal_lock_cbk, entrylk, dom, &loc, name,
+                    ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL);
+            break;
+        }
+    }
 
-	STACK_WIND (frame, sh_missing_entries_readlink_cbk,
-		    priv->children[sh->source],
-		    priv->children[sh->source]->fops->readlink,
-		    &local->loc, 4096);
+    loc_wipe(&loc);
 
-	return 0;
+    return afr_locked_fill(frame, this, locked_on);
 }
 
+int
+afr_selfheal_tie_breaker_entrylk(call_frame_t *frame, xlator_t *this,
+                                 inode_t *inode, char *dom, const char *name,
+                                 unsigned char *locked_on)
+{
+    loc_t loc = {
+        0,
+    };
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int lock_count = 0;
+    int eagain_count = 0;
 
-static int
-sh_missing_entries_create (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	int              type = 0;
-	int              i = 0;
-	afr_private_t   *priv = NULL;
-	int              enoent_count = 0;
-	int              govinda_gOvinda = 0;
-
-
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
-
-	for (i = 0; i < priv->child_count; i++) {
-                if (!local->child_up[i])
-                        continue;
-
-		if (sh->child_errno[i]) {
-			if (sh->child_errno[i] == ENOENT)
-				enoent_count++;
-		} else {
-			if (type) {
-				if (type != sh->buf[i].ia_type) {
-                                        gf_log (this->name, GF_LOG_TRACE,
-                                                "file %s is govinda!",
-                                                local->loc.path);
-
-					govinda_gOvinda = 1;
-                                }
-			} else {
-				sh->source = i;
-				type = sh->buf[i].ia_type;
-			}
-		}
-	}
-
-	if (govinda_gOvinda) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"conflicting filetypes exist for path %s. returning.",
-			local->loc.path);
-
-		local->govinda_gOvinda = 1;
-		sh_missing_entries_finish (frame, this);
-		return 0;
-	}
-
-	if (!type) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"no source found for %s. all nodes down?. returning.",
-			local->loc.path);
-		/* subvolumes down and/or file does not exist */
-		sh_missing_entries_finish (frame, this);
-		return 0;
-	}
-
-	if (enoent_count == 0) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"no missing files - %s. proceeding to metadata check",
-			local->loc.path);
-		/* proceed to next step - metadata self-heal */
-		sh_missing_entries_finish (frame, this);
-		return 0;
-	}
-
-	switch (type) {
-	case IA_IFSOCK:
-	case IA_IFREG:
-	case IA_IFBLK:
-	case IA_IFCHR:
-	case IA_IFIFO:
-		sh_missing_entries_mknod (frame, this);
-		break;
-	case IA_IFLNK:
-		sh_missing_entries_readlink (frame, this);
-		break;
-	case IA_IFDIR:
-		sh_missing_entries_mkdir (frame, this);
-		break;
-	default:
-		gf_log (this->name, GF_LOG_ERROR,
-			"unknown file type: 0%o", type);
-		local->govinda_gOvinda = 1;
-		sh_missing_entries_finish (frame, this);
-	}
-
-	return 0;
-}
+    priv = this->private;
+    local = frame->local;
 
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
 
-static int
-sh_missing_entries_lookup_cbk (call_frame_t *frame, void *cookie,
-			       xlator_t *this,
-			       int32_t op_ret, int32_t op_errno,
-                               inode_t *inode, struct iatt *buf, dict_t *xattr,
-                               struct iatt *postparent)
-{
-	int              child_index = 0;
-	afr_local_t     *local = NULL;
-	int              call_count = 0;
-	afr_private_t   *priv = NULL;
+    AFR_ONALL(frame, afr_selfheal_lock_cbk, entrylk, dom, &loc, name,
+              ENTRYLK_LOCK_NB, ENTRYLK_WRLCK, NULL);
 
+    afr_get_lock_and_eagain_counts(priv, local->replies, &lock_count,
+                                   &eagain_count);
 
-	local = frame->local;
-	priv = this->private;
+    if (lock_count > priv->child_count / 2 && eagain_count) {
+        afr_locked_fill(frame, this, locked_on);
+        afr_selfheal_unentrylk(frame, this, inode, dom, name, locked_on, NULL);
 
-	child_index = (long) cookie;
+        AFR_SEQ(frame, afr_selfheal_lock_cbk, entrylk, dom, &loc, name,
+                ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL);
+    }
 
-	LOCK (&frame->lock);
-	{
-		if (op_ret == 0) {
-			gf_log (this->name, GF_LOG_TRACE,
-				"path %s on subvolume %s is of mode 0%o",
-				local->loc.path,
-				priv->children[child_index]->name,
-				buf->ia_type);
+    loc_wipe(&loc);
 
-			local->self_heal.buf[child_index] = *buf;
-                        local->self_heal.parentbuf        = *postparent;
-		} else {
-			gf_log (this->name, GF_LOG_TRACE,
-				"path %s on subvolume %s => -1 (%s)",
-				local->loc.path,
-				priv->children[child_index]->name,
-				strerror (op_errno));
+    return afr_locked_fill(frame, this, locked_on);
+}
 
-			local->self_heal.child_errno[child_index] = op_errno;
-		}
+int
+afr_selfheal_unentrylk(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                       char *dom, const char *name, unsigned char *locked_on,
+                       dict_t *xdata)
+{
+    loc_t loc = {
+        0,
+    };
 
-	}
-	UNLOCK (&frame->lock);
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
 
-	call_count = afr_frame_return (frame);
+    AFR_ONLIST(locked_on, frame, afr_selfheal_lock_cbk, entrylk, dom, &loc,
+               name, ENTRYLK_UNLOCK, ENTRYLK_WRLCK, xdata);
 
-	if (call_count == 0) {
-		sh_missing_entries_create (frame, this);
-	}
+    loc_wipe(&loc);
 
-	return 0;
+    return 0;
 }
 
+gf_boolean_t
+afr_is_data_set(xlator_t *this, dict_t *xdata)
+{
+    return afr_is_pending_set(this, xdata, AFR_DATA_TRANSACTION);
+}
 
-static int
-sh_missing_entries_lookup (call_frame_t *frame, xlator_t *this)
+gf_boolean_t
+afr_is_metadata_set(xlator_t *this, dict_t *xdata)
 {
-	afr_local_t    *local = NULL;
-	int             i = 0;
-	int             call_count = 0;
-	afr_private_t  *priv = NULL;
-	dict_t         *xattr_req = NULL;
-	int             ret = -1;
+    return afr_is_pending_set(this, xdata, AFR_METADATA_TRANSACTION);
+}
 
-	local = frame->local;
-	priv  = this->private;
+gf_boolean_t
+afr_is_entry_set(xlator_t *this, dict_t *xdata)
+{
+    return afr_is_pending_set(this, xdata, AFR_ENTRY_TRANSACTION);
+}
 
-	call_count = afr_up_children_count (priv->child_count,
-                                            local->child_up);
+/*
+ * This function inspects the looked up replies (in an unlocked manner)
+ * and decides whether a locked verification and possible healing is
+ * required or not. It updates the three booleans for each type
+ * of healing. If the boolean flag gets set to FALSE, then we are sure
+ * no healing is required. If the boolean flag gets set to TRUE then
+ * we have to proceed with locked reinspection.
+ */
 
-	local->call_count = call_count;
+int
+afr_selfheal_unlocked_inspect(call_frame_t *frame, xlator_t *this, uuid_t gfid,
+                              inode_t **link_inode, gf_boolean_t *data_selfheal,
+                              gf_boolean_t *metadata_selfheal,
+                              gf_boolean_t *entry_selfheal,
+                              struct afr_reply *replies_dst)
+{
+    afr_private_t *priv = NULL;
+    inode_t *inode = NULL;
+    int i = 0;
+    int valid_cnt = 0;
+    struct iatt first = {
+        0,
+    };
+    int first_idx = 0;
+    struct afr_reply *replies = NULL;
+    int ret = -1;
+
+    priv = this->private;
+
+    inode = afr_inode_find(this, gfid);
+    if (!inode)
+        goto out;
+
+    replies = alloca0(sizeof(*replies) * priv->child_count);
+
+    ret = afr_selfheal_unlocked_discover(frame, inode, gfid, replies);
+    if (ret)
+        goto out;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid)
+            continue;
+        if (replies[i].op_ret == -1)
+            continue;
+
+        /* The data segment of the changelog can be non-zero to indicate
+         * the directory needs a full heal. So the check below ensures
+         * it's not a directory before setting the data_selfheal boolean.
+         */
+        if (data_selfheal && !IA_ISDIR(replies[i].poststat.ia_type) &&
+            afr_is_data_set(this, replies[i].xdata))
+            *data_selfheal = _gf_true;
 
-	xattr_req = dict_new();
+        if (metadata_selfheal && afr_is_metadata_set(this, replies[i].xdata))
+            *metadata_selfheal = _gf_true;
 
-	if (xattr_req) {
-                for (i = 0; i < priv->child_count; i++) {
-                        ret = dict_set_uint64 (xattr_req,
-                                               priv->pending_key[i],
-                                               3 * sizeof(int32_t));
-                        if (ret < 0)
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "Unable to set dict value.");
-                }
+        if (entry_selfheal && afr_is_entry_set(this, replies[i].xdata))
+            *entry_selfheal = _gf_true;
+
+        valid_cnt++;
+        if (valid_cnt == 1) {
+            first = replies[i].poststat;
+            first_idx = i;
+            continue;
         }
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (local->child_up[i]) {
-			gf_log (this->name, GF_LOG_TRACE,
-				"looking up %s on subvolume %s",
-				local->loc.path, priv->children[i]->name);
+        if (!IA_EQUAL(first, replies[i].poststat, type)) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
+                   "TYPE mismatch %d vs %d on %s for gfid:%s",
+                   (int)first.ia_type, (int)replies[i].poststat.ia_type,
+                   priv->children[i]->name,
+                   uuid_utoa(replies[i].poststat.ia_gfid));
+            gf_event(EVENT_AFR_SPLIT_BRAIN,
+                     "client-pid=%d;"
+                     "subvol=%s;"
+                     "type=file;gfid=%s;"
+                     "ia_type-%d=%s;ia_type-%d=%s",
+                     this->ctx->cmd_args.client_pid, this->name,
+                     uuid_utoa(replies[i].poststat.ia_gfid), first_idx,
+                     gf_inode_type_to_str(first.ia_type), i,
+                     gf_inode_type_to_str(replies[i].poststat.ia_type));
+            ret = -EIO;
+            goto out;
+        }
 
-			STACK_WIND_COOKIE (frame,
-					   sh_missing_entries_lookup_cbk,
-					   (void *) (long) i,
-					   priv->children[i],
-					   priv->children[i]->fops->lookup,
-					   &local->loc, xattr_req);
+        if (!IA_EQUAL(first, replies[i].poststat, uid)) {
+            gf_msg_debug(this->name, 0,
+                         "UID mismatch "
+                         "%d vs %d on %s for gfid:%s",
+                         (int)first.ia_uid, (int)replies[i].poststat.ia_uid,
+                         priv->children[i]->name,
+                         uuid_utoa(replies[i].poststat.ia_gfid));
 
-			if (!--call_count)
-				break;
-		}
-	}
+            if (metadata_selfheal)
+                *metadata_selfheal = _gf_true;
+        }
 
-	if (xattr_req)
-		dict_unref (xattr_req);
+        if (!IA_EQUAL(first, replies[i].poststat, gid)) {
+            gf_msg_debug(this->name, 0,
+                         "GID mismatch "
+                         "%d vs %d on %s for gfid:%s",
+                         (int)first.ia_uid, (int)replies[i].poststat.ia_uid,
+                         priv->children[i]->name,
+                         uuid_utoa(replies[i].poststat.ia_gfid));
 
-	return 0;
-}
+            if (metadata_selfheal)
+                *metadata_selfheal = _gf_true;
+        }
 
+        if (!IA_EQUAL(first, replies[i].poststat, prot)) {
+            gf_msg_debug(this->name, 0,
+                         "MODE mismatch "
+                         "%d vs %d on %s for gfid:%s",
+                         (int)st_mode_from_ia(first.ia_prot, 0),
+                         (int)st_mode_from_ia(replies[i].poststat.ia_prot, 0),
+                         priv->children[i]->name,
+                         uuid_utoa(replies[i].poststat.ia_gfid));
+
+            if (metadata_selfheal)
+                *metadata_selfheal = _gf_true;
+        }
 
+        if (IA_ISREG(first.ia_type) &&
+            !IA_EQUAL(first, replies[i].poststat, size)) {
+            gf_msg_debug(this->name, 0,
+                         "SIZE mismatch "
+                         "%lld vs %lld on %s for gfid:%s",
+                         (long long)first.ia_size,
+                         (long long)replies[i].poststat.ia_size,
+                         priv->children[i]->name,
+                         uuid_utoa(replies[i].poststat.ia_gfid));
+
+            if (data_selfheal)
+                *data_selfheal = _gf_true;
+        }
+    }
 
-int
-afr_sh_post_nonblocking_entrylk_cbk (call_frame_t *frame, xlator_t *this)
+    if (valid_cnt > 0 && link_inode) {
+        *link_inode = inode_link(inode, NULL, NULL, &first);
+        if (!*link_inode) {
+            ret = -EINVAL;
+            goto out;
+        }
+    } else if (valid_cnt < 2) {
+        ret = afr_check_stale_error(replies, priv);
+        goto out;
+    }
+
+    ret = 0;
+out:
+    if (replies && replies_dst)
+        afr_replies_copy(replies_dst, replies, priv->child_count);
+    if (inode)
+        inode_unref(inode);
+    if (replies)
+        afr_replies_wipe(replies, priv->child_count);
+
+    return ret;
+}
+
+inode_t *
+afr_inode_find(xlator_t *this, uuid_t gfid)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
+    inode_table_t *table = NULL;
+    inode_t *inode = NULL;
 
-        local    = frame->local;
-        int_lock = &local->internal_lock;
+    table = this->itable;
+    if (!table)
+        return NULL;
 
-        if (int_lock->lock_op_ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Non blocking entrylks failed.");
-                afr_sh_missing_entries_done (frame, this);
-        } else {
+    inode = inode_find(table, gfid);
+    if (inode)
+        return inode;
 
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Non blocking entrylks done. Proceeding to FOP");
-                sh_missing_entries_lookup (frame, this);
-        }
+    inode = inode_new(table);
+    if (!inode)
+        return NULL;
 
-        return 0;
+    gf_uuid_copy(inode->gfid, gfid);
+
+    return inode;
 }
 
-static int
-afr_sh_entrylk (call_frame_t *frame, xlator_t *this)
+call_frame_t *
+afr_frame_create(xlator_t *this, int32_t *op_errno)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_self_heal_t     *sh       = NULL;
+    call_frame_t *frame = NULL;
+    afr_local_t *local = NULL;
+    pid_t pid = GF_CLIENT_PID_SELF_HEALD;
 
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        sh       = &local->self_heal;
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame) {
+        if (op_errno)
+            *op_errno = ENOMEM;
+        return NULL;
+    }
 
-        int_lock->transaction_lk_type = AFR_SELFHEAL_LK;
-        int_lock->selfheal_lk_type    = AFR_ENTRY_SELF_HEAL_LK;
+    local = AFR_FRAME_INIT(frame, (*op_errno));
+    if (!local) {
+        STACK_DESTROY(frame->root);
+        return NULL;
+    }
 
-        afr_set_lock_number (frame, this);
+    syncopctx_setfspid(&pid);
 
-        int_lock->lk_basename = local->loc.name;
-        int_lock->lk_loc      = &sh->parent_loc;
-        int_lock->lock_cbk    = afr_sh_post_nonblocking_entrylk_cbk;
+    frame->root->pid = pid;
 
-        afr_nonblocking_entrylk (frame, this);
+    afr_set_lk_owner(frame, this, frame->root);
 
-        return 0;
+    return frame;
 }
 
-static int
-afr_self_heal_missing_entries (call_frame_t *frame, xlator_t *this)
+int
+afr_selfheal_newentry_mark(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                           int source, struct afr_reply *replies,
+                           unsigned char *sources, unsigned char *newentry)
 {
-        afr_internal_lock_t *int_lock = NULL;
-	afr_local_t         *local    = NULL;
-	afr_self_heal_t     *sh       = NULL;
-	afr_private_t       *priv     = NULL;
+    int ret = 0;
+    int i = 0;
+    afr_private_t *priv = NULL;
+    dict_t *xattr = NULL;
+    int **changelog = NULL;
 
-	local    = frame->local;
-        int_lock = &local->internal_lock;
-	sh       = &local->self_heal;
-	priv     = this->private;
+    priv = this->private;
 
-	gf_log (this->name, GF_LOG_TRACE,
-		"attempting to recreate missing entries for path=%s",
-		local->loc.path);
+    gf_uuid_copy(inode->gfid, replies[source].poststat.ia_gfid);
 
-	afr_build_parent_loc (&sh->parent_loc, &local->loc);
+    xattr = dict_new();
+    if (!xattr)
+        return -ENOMEM;
 
-        afr_sh_entrylk (frame, this);
-	return 0;
+    changelog = afr_mark_pending_changelog(priv, newentry, xattr,
+                                           replies[source].poststat.ia_type);
+
+    if (!changelog) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i])
+            continue;
+        ret |= afr_selfheal_post_op(frame, this, inode, i, xattr, NULL);
+    }
+out:
+    if (changelog)
+        afr_matrix_cleanup(changelog, priv->child_count);
+    if (xattr)
+        dict_unref(xattr);
+    return ret;
 }
 
-afr_local_t *afr_local_copy (afr_local_t *l, xlator_t *this)
+int
+afr_selfheal_do(call_frame_t *frame, xlator_t *this, uuid_t gfid)
 {
-        afr_private_t *priv = NULL;
-        afr_local_t   *lc     = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_self_heal_t *shc = NULL;
+    int ret = -1;
+    int entry_ret = 1;
+    int metadata_ret = 1;
+    int data_ret = 1;
+    int or_ret = 0;
+    inode_t *inode = NULL;
+    fd_t *fd = NULL;
+    gf_boolean_t data_selfheal = _gf_false;
+    gf_boolean_t metadata_selfheal = _gf_false;
+    gf_boolean_t entry_selfheal = _gf_false;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+
+    ret = afr_selfheal_unlocked_inspect(frame, this, gfid, &inode,
+                                        &data_selfheal, &metadata_selfheal,
+                                        &entry_selfheal, NULL);
+    if (ret)
+        goto out;
+
+    if (!(data_selfheal || metadata_selfheal || entry_selfheal)) {
+        ret = 2;
+        goto out;
+    }
+
+    if (inode->ia_type == IA_IFREG) {
+        ret = afr_selfheal_data_open(this, inode, &fd);
+        if (!fd) {
+            ret = -EIO;
+            goto out;
+        }
+    }
 
+    if (data_selfheal && priv->data_self_heal)
+        data_ret = afr_selfheal_data(frame, this, fd);
 
-        priv = this->private;
+    if (metadata_selfheal && priv->metadata_self_heal)
+        metadata_ret = afr_selfheal_metadata(frame, this, inode);
 
-        sh = &l->self_heal;
+    if (entry_selfheal && priv->entry_self_heal)
+        entry_ret = afr_selfheal_entry(frame, this, inode);
 
-        lc = GF_CALLOC (1, sizeof (afr_local_t),
-                        gf_afr_mt_afr_local_t);
+    or_ret = (data_ret | metadata_ret | entry_ret);
 
-        shc = &lc->self_heal;
+    if (data_ret == -EIO || metadata_ret == -EIO || entry_ret == -EIO)
+        ret = -EIO;
+    else if (data_ret == 1 && metadata_ret == 1 && entry_ret == 1)
+        ret = 1;
+    else if (or_ret < 0)
+        ret = or_ret;
+    else
+        ret = 0;
 
-        shc->unwind = sh->unwind;
-        shc->need_data_self_heal = sh->need_data_self_heal;
-        shc->need_metadata_self_heal = sh->need_metadata_self_heal;
-        shc->need_entry_self_heal = sh->need_entry_self_heal;
-        shc->forced_merge = sh->forced_merge;
-        shc->healing_fd_opened = sh->healing_fd_opened;
-        shc->data_lock_held = sh->data_lock_held;
-        if (sh->healing_fd && !sh->healing_fd_opened)
-                shc->healing_fd = fd_ref (sh->healing_fd);
-        else
-                shc->healing_fd = sh->healing_fd;
-        shc->background = sh->background;
-        shc->type = sh->type;
-
-        if (l->loc.path)
-                loc_copy (&lc->loc, &l->loc);
-
-        lc->child_up  = memdup (l->child_up, priv->child_count);
-        if (l->xattr_req)
-                lc->xattr_req = dict_ref (l->xattr_req);
-
-        if (l->cont.lookup.inode)
-                lc->cont.lookup.inode = inode_ref (l->cont.lookup.inode);
-        if (l->cont.lookup.xattr)
-                lc->cont.lookup.xattr = dict_ref (l->cont.lookup.xattr);
-        if (l->internal_lock.inode_locked_nodes)
-                lc->internal_lock.inode_locked_nodes =
-                        memdup (l->internal_lock.inode_locked_nodes,
-                                priv->child_count);
-        else
-                lc->internal_lock.inode_locked_nodes =
-                        GF_CALLOC (sizeof (*l->internal_lock.inode_locked_nodes),
-                                   priv->child_count,
-                                   gf_afr_mt_char);
-        if (l->internal_lock.entry_locked_nodes)
-                lc->internal_lock.entry_locked_nodes =
-                        memdup (l->internal_lock.entry_locked_nodes,
-                                priv->child_count);
-        else
-                lc->internal_lock.entry_locked_nodes =
-                        GF_CALLOC (sizeof (*l->internal_lock.entry_locked_nodes),
-                                   priv->child_count,
-                                   gf_afr_mt_char);
-        if (l->internal_lock.locked_nodes)
-                lc->internal_lock.locked_nodes =
-                        memdup (l->internal_lock.locked_nodes,
-                                priv->child_count);
-        else
-                lc->internal_lock.locked_nodes =
-                        GF_CALLOC (sizeof (*l->internal_lock.locked_nodes),
-                                   priv->child_count,
-                                   gf_afr_mt_char);
+out:
+    if (inode)
+        inode_unref(inode);
+    if (fd)
+        fd_unref(fd);
+    return ret;
+}
+/*
+ * This is the entry point for healing a given GFID. The return values for this
+ * function are as follows:
+ * '0' if the self-heal is successful
+ * '1' if the afr-xattrs are non-zero (due to on-going IO) and no heal is needed
+ * '2' if the afr-xattrs are all-zero and no heal is needed
+ * $errno if the heal on the gfid failed.
+ */
 
-        lc->internal_lock.inodelk_lock_count =
-                l->internal_lock.inodelk_lock_count;
-        lc->internal_lock.entrylk_lock_count =
-                l->internal_lock.entrylk_lock_count;
+int
+afr_selfheal(xlator_t *this, uuid_t gfid)
+{
+    int ret = -1;
+    call_frame_t *frame = NULL;
+    afr_local_t *local = NULL;
 
-        return lc;
-}
+    frame = afr_frame_create(this, NULL);
+    if (!frame)
+        return ret;
 
+    local = frame->local;
+    local->xdata_req = dict_new();
 
-int
-afr_self_heal_completion_cbk (call_frame_t *bgsh_frame, xlator_t *this)
-{
-        afr_private_t *   priv  = NULL;
-	afr_local_t *     local = NULL;
-        afr_self_heal_t * sh    = NULL;
-        char              sh_type_str[256] = {0,};
-
-        priv  = this->private;
-	local = bgsh_frame->local;
-        sh    = &local->self_heal;
-
-	if (local->govinda_gOvinda) {
-                afr_set_split_brain (this, local->cont.lookup.inode, _gf_true);
-	} else {
-                afr_set_split_brain (this, local->cont.lookup.inode, _gf_false);
-	}
-
-        afr_self_heal_type_str_get(sh, sh_type_str,
-                                   sizeof(sh_type_str));
-        gf_log (this->name, GF_LOG_NORMAL,
-                "background %s self-heal completed on %s", sh_type_str,
-                local->loc.path);
-
-        if (!sh->unwound) {
-                sh->unwind (sh->orig_frame, this);
-        }
+    ret = afr_selfheal_do(frame, this, gfid);
 
-        if (sh->background) {
-                LOCK (&priv->lock);
-                {
-                        priv->background_self_heals_started--;
-                }
-                UNLOCK (&priv->lock);
-        }
+    if (frame)
+        AFR_STACK_DESTROY(frame);
 
-        AFR_STACK_DESTROY (bgsh_frame);
+    return ret;
+}
 
-	return 0;
+afr_local_t *
+__afr_dequeue_heals(afr_private_t *priv)
+{
+    afr_local_t *local = NULL;
+
+    if (list_empty(&priv->heal_waiting))
+        goto none;
+    if ((priv->background_self_heal_count > 0) &&
+        (priv->healers >= priv->background_self_heal_count))
+        goto none;
+
+    local = list_entry(priv->heal_waiting.next, afr_local_t, healer);
+    priv->heal_waiters--;
+    GF_ASSERT(priv->heal_waiters >= 0);
+    list_del_init(&local->healer);
+    list_add(&local->healer, &priv->healing);
+    priv->healers++;
+    return local;
+none:
+    gf_msg_debug(THIS->name, 0,
+                 "Nothing dequeued. "
+                 "Num healers: %d, Num Waiters: %d",
+                 priv->healers, priv->heal_waiters);
+    return NULL;
 }
 
 int
-afr_self_heal (call_frame_t *frame, xlator_t *this)
+afr_refresh_selfheal_wrap(void *opaque)
 {
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-	int              i = 0;
+    call_frame_t *heal_frame = opaque;
+    afr_local_t *local = heal_frame->local;
+    int ret = 0;
 
-        call_frame_t *sh_frame = NULL;
-        afr_local_t  *sh_local = NULL;
+    ret = afr_selfheal(heal_frame->this, local->refreshinode->gfid);
+    return ret;
+}
 
+int
+afr_refresh_heal_done(int ret, call_frame_t *frame, void *opaque)
+{
+    call_frame_t *heal_frame = opaque;
+    xlator_t *this = heal_frame->this;
+    afr_private_t *priv = this->private;
+    afr_local_t *local = heal_frame->local;
+
+    LOCK(&priv->lock);
+    {
+        list_del_init(&local->healer);
+        priv->healers--;
+        GF_ASSERT(priv->healers >= 0);
+        local = __afr_dequeue_heals(priv);
+    }
+    UNLOCK(&priv->lock);
+
+    AFR_STACK_DESTROY(heal_frame);
+
+    if (local)
+        afr_heal_synctask(this, local);
+    return 0;
+}
 
-	local = frame->local;
-	priv  = this->private;
+void
+afr_heal_synctask(xlator_t *this, afr_local_t *local)
+{
+    int ret = 0;
+    call_frame_t *heal_frame = NULL;
+
+    heal_frame = local->heal_frame;
+    ret = synctask_new(this->ctx->env, afr_refresh_selfheal_wrap,
+                       afr_refresh_heal_done, heal_frame, heal_frame);
+    if (ret < 0)
+        /* Heal not launched. Will be queued when the next inode
+         * refresh happens and shd hasn't healed it yet. */
+        afr_refresh_heal_done(ret, heal_frame, heal_frame);
+}
 
-        afr_set_lk_owner (frame, this);
+gf_boolean_t
+afr_throttled_selfheal(call_frame_t *frame, xlator_t *this)
+{
+    gf_boolean_t can_heal = _gf_true;
+    afr_private_t *priv = this->private;
+    afr_local_t *local = frame->local;
+
+    LOCK(&priv->lock);
+    {
+        if ((priv->background_self_heal_count > 0) &&
+            (priv->heal_wait_qlen + priv->background_self_heal_count) >
+                (priv->heal_waiters + priv->healers)) {
+            list_add_tail(&local->healer, &priv->heal_waiting);
+            priv->heal_waiters++;
+            local = __afr_dequeue_heals(priv);
+        } else {
+            can_heal = _gf_false;
+        }
+    }
+    UNLOCK(&priv->lock);
 
-        if (local->self_heal.background) {
-                LOCK (&priv->lock);
-                {
-                        if (priv->background_self_heals_started
-                            > priv->background_self_heal_count) {
+    if (can_heal) {
+        if (local)
+            afr_heal_synctask(this, local);
+        else
+            gf_msg_debug(this->name, 0,
+                         "Max number of heals are "
+                         "pending, background self-heal rejected.");
+    }
 
-                                local->self_heal.background = _gf_false;
+    return can_heal;
+}
 
-                        } else {
-                                priv->background_self_heals_started++;
-                        }
-                }
-                UNLOCK (&priv->lock);
+int
+afr_choose_source_by_policy(afr_private_t *priv, unsigned char *sources,
+                            afr_transaction_type type)
+{
+    int source = -1;
+    int i = 0;
+
+    /* Give preference to local child to save on bandwidth */
+    for (i = 0; i < priv->child_count; i++) {
+        if (priv->local[i] && sources[i]) {
+            if ((type == AFR_DATA_TRANSACTION) && AFR_IS_ARBITER_BRICK(priv, i))
+                continue;
+
+            source = i;
+            goto out;
         }
+    }
 
-        gf_log (this->name, GF_LOG_TRACE,
-                "performing self heal on %s (metadata=%d data=%d entry=%d)",
-                local->loc.path,
-                local->self_heal.need_metadata_self_heal,
-                local->self_heal.need_data_self_heal,
-                local->self_heal.need_entry_self_heal);
-
-        sh_frame        = copy_frame (frame);
-        sh_local        = afr_local_copy (local, this);
-        sh_frame->local = sh_local;
-        sh              = &sh_local->self_heal;
-
-        sh->orig_frame  = frame;
-
-        sh->completion_cbk = afr_self_heal_completion_cbk;
-
-
-	sh->buf = GF_CALLOC (priv->child_count, sizeof (struct iatt),
-                             gf_afr_mt_iatt);
-	sh->child_errno = GF_CALLOC (priv->child_count, sizeof (int),
-                                     gf_afr_mt_int);
-	sh->success = GF_CALLOC (priv->child_count, sizeof (int),
-                                gf_afr_mt_int);
-	sh->xattr = GF_CALLOC (priv->child_count, sizeof (dict_t *),
-                                gf_afr_mt_dict_t);
-	sh->sources = GF_CALLOC (sizeof (*sh->sources), priv->child_count,
-                                gf_afr_mt_int);
-	sh->locked_nodes = GF_CALLOC (sizeof (*sh->locked_nodes), 
-                                      priv->child_count,
-                                      gf_afr_mt_int);
-
-	sh->pending_matrix = GF_CALLOC (sizeof (int32_t *), priv->child_count,
-                                        gf_afr_mt_int32_t);
-
-	for (i = 0; i < priv->child_count; i++) {
-		sh->pending_matrix[i] = GF_CALLOC (sizeof (int32_t),
-					  	   priv->child_count,
-                                                   gf_afr_mt_int32_t);
-	}
-
-	sh->delta_matrix = GF_CALLOC (sizeof (int32_t *), priv->child_count,
-                                      gf_afr_mt_int32_t);
-	for (i = 0; i < priv->child_count; i++) {
-		sh->delta_matrix[i] = GF_CALLOC (sizeof (int32_t),
-					         priv->child_count,
-                                                 gf_afr_mt_int32_t);
-	}
-
-	if (local->success_count && local->enoent_count) {
-		afr_self_heal_missing_entries (sh_frame, this);
-	} else {
-		gf_log (this->name, GF_LOG_TRACE,
-			"proceeding to metadata check on %s",
-			local->loc.path);
-
-		afr_sh_missing_entries_done (sh_frame, this);
-	}
-
-	return 0;
+    for (i = 0; i < priv->child_count; i++) {
+        if (sources[i]) {
+            source = i;
+            goto out;
+        }
+    }
+out:
+    return source;
 }
 
-void
-afr_self_heal_type_str_get (afr_self_heal_t *self_heal_p, char *str,
-                            size_t size)
+static int
+afr_anon_inode_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, inode_t *inode,
+                         struct iatt *buf, struct iatt *preparent,
+                         struct iatt *postparent, dict_t *xdata)
 {
-        GF_ASSERT (str && (size > 0));
+    afr_local_t *local = frame->local;
+    int i = (long)cookie;
+
+    local->replies[i].valid = 1;
+    local->replies[i].op_ret = op_ret;
+    local->replies[i].op_errno = op_errno;
+    if (op_ret == 0) {
+        local->op_ret = 0;
+        local->replies[i].poststat = *buf;
+        local->replies[i].preparent = *preparent;
+        local->replies[i].postparent = *postparent;
+    }
+    if (xdata) {
+        local->replies[i].xdata = dict_ref(xdata);
+    }
+
+    syncbarrier_wake(&local->barrier);
+    return 0;
+}
 
-        if (self_heal_p->need_metadata_self_heal) {
-                snprintf(str, size, " meta-data");
+int
+afr_anon_inode_create(xlator_t *this, int child, inode_t **linked_inode)
+{
+    call_frame_t *frame = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = this->private;
+    unsigned char *mkdir_on = alloca0(priv->child_count);
+    unsigned char *lookup_on = alloca0(priv->child_count);
+    loc_t loc = {0};
+    int32_t op_errno = 0;
+    int32_t child_op_errno = 0;
+    struct iatt iatt = {0};
+    dict_t *xdata = NULL;
+    uuid_t anon_inode_gfid = {0};
+    int mkdir_count = 0;
+    int i = 0;
+
+    /*Try to mkdir everywhere and return success if the dir exists on 'child'
+     */
+
+    if (!priv->use_anon_inode) {
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    frame = afr_frame_create(this, &op_errno);
+    if (op_errno) {
+        goto out;
+    }
+    local = frame->local;
+    if (!local->child_up[child]) {
+        /*Other bricks may need mkdir so don't error out yet*/
+        child_op_errno = ENOTCONN;
+    }
+    gf_uuid_parse(priv->anon_gfid_str, anon_inode_gfid);
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->child_up[i])
+            continue;
+
+        if (priv->anon_inode[i]) {
+            mkdir_on[i] = 0;
+        } else {
+            mkdir_on[i] = 1;
+            mkdir_count++;
         }
+    }
 
-        if (self_heal_p->need_data_self_heal) {
-                snprintf(str + strlen(str), size - strlen(str),
-                         " data");
+    if (mkdir_count == 0) {
+        *linked_inode = inode_find(this->itable, anon_inode_gfid);
+        if (*linked_inode) {
+            op_errno = 0;
+            goto out;
+        }
+    }
+
+    loc.parent = inode_ref(this->itable->root);
+    loc.name = priv->anon_inode_name;
+    loc.inode = inode_new(this->itable);
+    if (!loc.inode) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    xdata = dict_new();
+    if (!xdata) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    op_errno = -dict_set_gfuuid(xdata, "gfid-req", anon_inode_gfid, _gf_true);
+    if (op_errno) {
+        goto out;
+    }
+
+    if (mkdir_count == 0) {
+        memcpy(lookup_on, local->child_up, priv->child_count);
+        goto lookup;
+    }
+
+    AFR_ONLIST(mkdir_on, frame, afr_anon_inode_mkdir_cbk, mkdir, &loc, 0755, 0,
+               xdata);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!mkdir_on[i]) {
+            continue;
         }
 
-        if (self_heal_p->need_entry_self_heal) {
-                snprintf(str + strlen(str), size - strlen(str),
-                         " entry");
+        if (local->replies[i].op_ret == 0) {
+            priv->anon_inode[i] = 1;
+            iatt = local->replies[i].poststat;
+        } else if (local->replies[i].op_ret < 0 &&
+                   local->replies[i].op_errno == EEXIST) {
+            lookup_on[i] = 1;
+        } else if (i == child) {
+            child_op_errno = local->replies[i].op_errno;
+        }
+    }
+
+    if (AFR_COUNT(lookup_on, priv->child_count) == 0) {
+        goto link;
+    }
+
+lookup:
+    AFR_ONLIST(lookup_on, frame, afr_selfheal_discover_cbk, lookup, &loc,
+               xdata);
+    for (i = 0; i < priv->child_count; i++) {
+        if (!lookup_on[i]) {
+            continue;
         }
+
+        if (local->replies[i].op_ret == 0) {
+            if (gf_uuid_compare(anon_inode_gfid,
+                                local->replies[i].poststat.ia_gfid) == 0) {
+                priv->anon_inode[i] = 1;
+                iatt = local->replies[i].poststat;
+            } else {
+                if (i == child)
+                    child_op_errno = EINVAL;
+                gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_INVALID_DATA,
+                       "%s has gfid: %s", priv->anon_inode_name,
+                       uuid_utoa(local->replies[i].poststat.ia_gfid));
+            }
+        } else if (i == child) {
+            child_op_errno = local->replies[i].op_errno;
+        }
+    }
+link:
+    if (!gf_uuid_is_null(iatt.ia_gfid)) {
+        *linked_inode = inode_link(loc.inode, loc.parent, loc.name, &iatt);
+        if (*linked_inode) {
+            op_errno = 0;
+            inode_lookup(*linked_inode);
+        } else {
+            op_errno = ENOMEM;
+        }
+        goto out;
+    }
+
+out:
+    if (xdata)
+        dict_unref(xdata);
+    loc_wipe(&loc);
+    /*child_op_errno takes precedence*/
+    if (child_op_errno == 0) {
+        child_op_errno = op_errno;
+    }
+
+    if (child_op_errno && *linked_inode) {
+        inode_unref(*linked_inode);
+        *linked_inode = NULL;
+    }
+    if (frame)
+        AFR_STACK_DESTROY(frame);
+    return -child_op_errno;
 }
diff --git a/xlators/cluster/afr/src/afr-self-heal-common.h b/xlators/cluster/afr/src/afr-self-heal-common.h
deleted file mode 100644
index 6431feaff35..00000000000
--- a/xlators/cluster/afr/src/afr-self-heal-common.h
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
-  Copyright (c) 2008-2010 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef __AFR_SELF_HEAL_COMMON_H__
-#define __AFR_SELF_HEAL_COMMON_H__
-
-#define FILE_HAS_HOLES(buf) (((buf)->ia_size) > ((buf)->ia_blocks * 512))
-
-typedef enum {
-        AFR_SELF_HEAL_ENTRY,
-        AFR_SELF_HEAL_METADATA,
-        AFR_SELF_HEAL_DATA,
-} afr_self_heal_type;
-
-int
-afr_sh_select_source (int sources[], int child_count);
-
-int
-afr_sh_sink_count (int sources[], int child_count);
-
-int
-afr_sh_source_count (int sources[], int child_count);
-
-int
-afr_sh_supress_errenous_children (int sources[], int child_errno[],
-				  int child_count);
-
-void
-afr_sh_print_pending_matrix (int32_t *pending_matrix[], xlator_t *this);
-
-void
-afr_sh_build_pending_matrix (afr_private_t *priv,
-                             int32_t *pending_matrix[], dict_t *xattr[],
-			     int child_count, afr_transaction_type type);
-
-void
-afr_sh_pending_to_delta (afr_private_t *priv, dict_t **xattr,
-                         int32_t *delta_matrix[], int success[],
-                         int child_count, afr_transaction_type type);
-
-int
-afr_sh_mark_sources (afr_self_heal_t *sh, int child_count,
-                     afr_self_heal_type type);
-
-int
-afr_sh_delta_to_xattr (afr_private_t *priv,
-                       int32_t *delta_matrix[], dict_t *xattr[],
-		       int child_count, afr_transaction_type type);
-
-int
-afr_sh_is_matrix_zero (int32_t *pending_matrix[], int child_count);
-
-void
-afr_self_heal_type_str_get (afr_self_heal_t *self_heal_p, char *str,
-                            size_t size);
-
-#endif /* __AFR_SELF_HEAL_COMMON_H__ */
diff --git a/xlators/cluster/afr/src/afr-self-heal-data.c b/xlators/cluster/afr/src/afr-self-heal-data.c
index 0fd8dae69f9..37bcc2b3f9e 100644
--- a/xlators/cluster/afr/src/afr-self-heal-data.c
+++ b/xlators/cluster/afr/src/afr-self-heal-data.c
@@ -1,1113 +1,891 @@
 /*
-   Copyright (c) 2008-2010 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-#include <libgen.h>
-#include <unistd.h>
-#include <fnmatch.h>
-#include <sys/time.h>
-#include <stdlib.h>
-#include <signal.h>
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
 
-#include "glusterfs.h"
 #include "afr.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "stack.h"
-#include "list.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include "compat-errno.h"
-#include "compat.h"
-#include "byte-order.h"
-
-#include "afr-transaction.h"
 #include "afr-self-heal.h"
-#include "afr-self-heal-common.h"
-#include "afr-self-heal-algorithm.h"
-
+#include <glusterfs/byte-order.h>
+#include "protocol-common.h"
+#include "afr-messages.h"
+#include <glusterfs/events.h>
 
-int
-afr_sh_data_done (call_frame_t *frame, xlator_t *this)
+#define HAS_HOLES(i) ((i->ia_blocks * 512) < (i->ia_size))
+static int
+__checksum_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, uint32_t weak, uint8_t *strong, dict_t *xdata)
 {
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
-
-	/* 
-	   TODO: cleanup sh->* 
-	 */
-
-        if (sh->healing_fd && !sh->healing_fd_opened) {
-                /* unref only if we created the fd ourselves */
-
-		fd_unref (sh->healing_fd);
-		sh->healing_fd = NULL;
+    afr_local_t *local = NULL;
+    struct afr_reply *replies = NULL;
+    int i = (long)cookie;
+
+    local = frame->local;
+    replies = local->replies;
+
+    replies[i].valid = 1;
+    replies[i].op_ret = op_ret;
+    replies[i].op_errno = op_errno;
+    if (xdata) {
+        replies[i].buf_has_zeroes = dict_get_str_boolean(
+            xdata, "buf-has-zeroes", _gf_false);
+        replies[i].fips_mode_rchecksum = dict_get_str_boolean(
+            xdata, "fips-mode-rchecksum", _gf_false);
+    }
+    if (strong) {
+        if (replies[i].fips_mode_rchecksum) {
+            memcpy(local->replies[i].checksum, strong, SHA256_DIGEST_LENGTH);
+        } else {
+            memcpy(local->replies[i].checksum, strong, MD5_DIGEST_LENGTH);
         }
+    }
 
-/*         for (i = 0; i < priv->child_count; i++) */
-/*                 sh->locked_nodes[i] = 0; */
-
-	gf_log (this->name, GF_LOG_TRACE,
-		"self heal of %s completed",
-		local->loc.path);
-
-	sh->completion_cbk (frame, this);
-
-	return 0;
+    syncbarrier_wake(&local->barrier);
+    return 0;
 }
 
-
-int
-afr_sh_data_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		       int32_t op_ret, int32_t op_errno)
+static gf_boolean_t
+__afr_can_skip_data_block_heal(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                               int source, unsigned char *healed_sinks,
+                               off_t offset, size_t size, struct iatt *poststat)
 {
-	afr_local_t     *local = NULL;
-	afr_private_t   *priv  = NULL;
-	int              call_count = 0;
-
-        int child_index = (long) cookie;
-
-	local = frame->local;
-	priv = this->private;
-
-	LOCK (&frame->lock);
-	{
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "flush or setattr failed on %s on subvolume %s: %s",
-                                local->loc.path, priv->children[child_index]->name,
-                                strerror (op_errno));
-                }
-	}
-	UNLOCK (&frame->lock);
-
-	call_count = afr_frame_return (frame);
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    unsigned char *wind_subvols = NULL;
+    gf_boolean_t checksum_match = _gf_true;
+    struct afr_reply *replies = NULL;
+    dict_t *xdata = NULL;
+    int i = 0;
+
+    priv = this->private;
+    local = frame->local;
+    replies = local->replies;
+
+    xdata = dict_new();
+    if (!xdata)
+        goto out;
+    if (dict_set_int32_sizen(xdata, "check-zero-filled", 1)) {
+        dict_unref(xdata);
+        goto out;
+    }
+
+    wind_subvols = alloca0(priv->child_count);
+    for (i = 0; i < priv->child_count; i++) {
+        if (i == source || healed_sinks[i])
+            wind_subvols[i] = 1;
+    }
+
+    AFR_ONLIST(wind_subvols, frame, __checksum_cbk, rchecksum, fd, offset, size,
+               xdata);
+    if (xdata)
+        dict_unref(xdata);
+
+    if (!replies[source].valid || replies[source].op_ret != 0)
+        return _gf_false;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (i == source)
+            continue;
+        if (replies[i].valid) {
+            if (memcmp(replies[source].checksum, replies[i].checksum,
+                       replies[source].fips_mode_rchecksum
+                           ? SHA256_DIGEST_LENGTH
+                           : MD5_DIGEST_LENGTH)) {
+                checksum_match = _gf_false;
+                break;
+            }
+        }
+    }
 
-	if (call_count == 0) {
-		afr_sh_data_done (frame, this);
-	}
+    if (checksum_match) {
+        if (HAS_HOLES(poststat))
+            return _gf_true;
 
-	return 0;
+        /* For non-sparse files, we might be better off writing the
+         * zeroes to sinks to avoid mismatch of disk-usage in bricks. */
+        if (local->replies[source].buf_has_zeroes)
+            return _gf_false;
+        else
+            return _gf_true;
+    }
+out:
+    return _gf_false;
 }
 
-
-int
-afr_sh_data_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, struct iatt *statpre, struct iatt *statpost)
+static gf_boolean_t
+__afr_is_sink_zero_filled(xlator_t *this, fd_t *fd, size_t size, off_t offset,
+                          int sink)
 {
-        afr_sh_data_flush_cbk (frame, cookie, this, op_ret, op_errno);
-
-        return 0;
+    afr_private_t *priv = NULL;
+    struct iobref *iobref = NULL;
+    struct iovec *iovec = NULL;
+    int count = 0;
+    int ret = 0;
+    gf_boolean_t zero_filled = _gf_false;
+
+    priv = this->private;
+    ret = syncop_readv(priv->children[sink], fd, size, offset, 0, &iovec,
+                       &count, &iobref, NULL, NULL, NULL);
+    if (ret < 0)
+        goto out;
+    ret = iov_0filled(iovec, count);
+    if (!ret)
+        zero_filled = _gf_true;
+out:
+    if (iovec)
+        GF_FREE(iovec);
+    if (iobref)
+        iobref_unref(iobref);
+    return zero_filled;
 }
 
-
-int
-afr_sh_data_close (call_frame_t *frame, xlator_t *this)
+static int
+__afr_selfheal_data_read_write(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                               int source, unsigned char *healed_sinks,
+                               off_t offset, size_t size,
+                               struct afr_reply *replies, int type)
 {
-	afr_local_t     *local = NULL;
-	afr_private_t   *priv  = NULL;
-	afr_self_heal_t *sh    = NULL;
-
-	int  i            = 0;
-	int  call_count   = 0;
-        int  source       = 0;
-        int32_t valid     = 0;
-
-        struct iatt stbuf = {0,};
-
-        local = frame->local;
-        sh    = &local->self_heal;
-        priv  = this->private;
-
-        source       = sh->source;
-
-        valid |= (GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME);
-
-	stbuf.ia_atime = sh->buf[source].ia_atime;
-	stbuf.ia_atime_nsec = sh->buf[source].ia_atime_nsec;
-	stbuf.ia_mtime = sh->buf[source].ia_mtime;
-	stbuf.ia_mtime_nsec = sh->buf[source].ia_mtime_nsec;
-
-        if (sh->healing_fd_opened) {
-                /* not our job to close the fd */
+    struct iovec *iovec = NULL;
+    int count = 0;
+    struct iobref *iobref = NULL;
+    int ret = 0;
+    int i = 0;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+
+    ret = syncop_readv(priv->children[source], fd, size, offset, 0, &iovec,
+                       &count, &iobref, NULL, NULL, NULL);
+    if (ret <= 0)
+        return ret;
 
-                afr_sh_data_done (frame, this);
-                return 0;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!healed_sinks[i])
+            continue;
+
+            /*
+             * TODO: Use fiemap() and discard() to heal holes
+             * in the future.
+             *
+             * For now,
+             *
+             * - if the source had any holes at all,
+             * AND
+             * - if we are writing past the original file size
+             *   of the sink
+             * AND
+             * - is NOT the last block of the source file. if
+             *   the block contains EOF, it has to be written
+             *   in order to set the file size even if the
+             *   last block is 0-filled.
+             * AND
+             * - if the read buffer is filled with only 0's
+             *
+             * then, skip writing to this source. We don't depend
+             * on the write to happen to update the size as we
+             * have performed an ftruncate() upfront anyways.
+             */
+#define is_last_block(o, b, s) ((s >= o) && (s <= (o + b)))
+        if (HAS_HOLES((&replies[source].poststat)) &&
+            offset >= replies[i].poststat.ia_size &&
+            !is_last_block(offset, size, replies[source].poststat.ia_size) &&
+            (iov_0filled(iovec, count) == 0))
+            continue;
+
+        /* Avoid filling up sparse regions of the sink with 0-filled
+         * writes.*/
+        if (type == AFR_SELFHEAL_DATA_FULL &&
+            HAS_HOLES((&replies[source].poststat)) &&
+            ((offset + size) <= replies[i].poststat.ia_size) &&
+            (iov_0filled(iovec, count) == 0) &&
+            __afr_is_sink_zero_filled(this, fd, size, offset, i)) {
+            continue;
         }
 
-	if (!sh->healing_fd) {
-		afr_sh_data_done (frame, this);
-		return 0;
-	}
-
-	call_count        = (sh->active_sinks + 1) * 2;
-	local->call_count = call_count;
-
-	/* closed source */
-	gf_log (this->name, GF_LOG_TRACE,
-		"closing fd of %s on %s",
-		local->loc.path, priv->children[sh->source]->name);
-
-	STACK_WIND_COOKIE (frame, afr_sh_data_flush_cbk,
-			   (void *) (long) sh->source,
-			   priv->children[sh->source],
-			   priv->children[sh->source]->fops->flush,
-			   sh->healing_fd);
-	call_count--;
-
-        STACK_WIND_COOKIE (frame, afr_sh_data_setattr_cbk,
-                           (void *) (long) sh->source,
-                           priv->children[sh->source],
-                           priv->children[sh->source]->fops->setattr,
-                           &local->loc, &stbuf, valid);
-
-        call_count--;
-
-        if (call_count == 0)
-                return 0;
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (sh->sources[i] || !local->child_up[i])
-			continue;
-
-		gf_log (this->name, GF_LOG_TRACE,
-			"closing fd of %s on %s",
-			local->loc.path, priv->children[i]->name);
-
-		STACK_WIND_COOKIE (frame, afr_sh_data_flush_cbk,
-				   (void *) (long) i,
-				   priv->children[i],
-				   priv->children[i]->fops->flush,
-				   sh->healing_fd);
-
-                call_count--;
-
-                STACK_WIND_COOKIE (frame, afr_sh_data_setattr_cbk,
-				   (void *) (long) i,
-				   priv->children[i],
-				   priv->children[i]->fops->setattr,
-				   &local->loc, &stbuf, valid);
-
-		if (!--call_count)
-			break;
-	}
-
-	return 0;
-}
-
+        ret = syncop_writev(priv->children[i], fd, iovec, count, offset, iobref,
+                            0, NULL, NULL, NULL, NULL);
+        if (ret != iov_length(iovec, count)) {
+            /* write() failed on this sink. unset the corresponding
+               member in sinks[] (which is healed_sinks[] in the
+               caller) so that this server does NOT get considered
+               as successfully healed.
+            */
+            healed_sinks[i] = 0;
+        }
+    }
+    if (iovec)
+        GF_FREE(iovec);
+    if (iobref)
+        iobref_unref(iobref);
 
-int
-afr_sh_data_unlck_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		       int32_t op_ret, int32_t op_errno)
-{
-	afr_local_t * local = NULL;
-	int           call_count = 0;
-	int           child_index = (long) cookie;
-
-	
-	local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret == -1) {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"locking inode of %s on child %d failed: %s",
-				local->loc.path, child_index,
-				strerror (op_errno));
-		} else {
-			gf_log (this->name, GF_LOG_TRACE,
-				"inode of %s on child %d locked",
-				local->loc.path, child_index);
-		}
-	}
-	UNLOCK (&frame->lock);
-
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0) {
-		afr_sh_data_close (frame, this);
-	}
-
-	return 0;
+    return ret;
 }
 
-
-int
-afr_sh_data_unlock (call_frame_t *frame, xlator_t *this)
+static gf_boolean_t
+afr_source_sinks_locked(xlator_t *this, unsigned char *locked_on, int source,
+                        unsigned char *healed_sinks)
 {
-        afr_local_t         *local    = NULL;
-        afr_internal_lock_t *int_lock = NULL;
-	afr_self_heal_t     *sh       = NULL;
+    afr_private_t *priv = this->private;
+    int i = 0;
 
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-	sh       = &local->self_heal;
+    if (!locked_on[source])
+        return _gf_false;
 
-        GF_ASSERT (!sh->data_lock_held);
+    for (i = 0; i < priv->child_count; i++) {
+        if (healed_sinks[i] && locked_on[i])
+            return _gf_true;
+    }
 
-        int_lock->lock_cbk = afr_sh_data_close;
-        afr_unlock (frame, this);
-
-	return 0;
+    return _gf_false;
 }
 
-
-int
-afr_sh_data_finish (call_frame_t *frame, xlator_t *this)
+static int
+afr_selfheal_data_block(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                        int source, unsigned char *healed_sinks, off_t offset,
+                        size_t size, int type, struct afr_reply *replies)
 {
-	afr_local_t   *local = NULL;
-	afr_self_heal_t *sh = NULL;
-
-	local = frame->local;
-	sh = &local->self_heal;
-
-	gf_log (this->name, GF_LOG_TRACE,
-		"finishing data selfheal of %s", local->loc.path);
+    int ret = -1;
+    afr_private_t *priv = NULL;
+    unsigned char *data_lock = NULL;
+
+    priv = this->private;
+    data_lock = alloca0(priv->child_count);
+
+    ret = afr_selfheal_inodelk(frame, this, fd->inode, this->name, offset, size,
+                               data_lock);
+    {
+        if (!afr_source_sinks_locked(this, data_lock, source, healed_sinks)) {
+            ret = -ENOTCONN;
+            goto unlock;
+        }
 
-        if (!sh->data_lock_held)
-                afr_sh_data_unlock (frame, this);
-        else
-                afr_sh_data_close (frame, this);
+        if (type == AFR_SELFHEAL_DATA_DIFF &&
+            __afr_can_skip_data_block_heal(frame, this, fd, source,
+                                           healed_sinks, offset, size,
+                                           &replies[source].poststat)) {
+            ret = 0;
+            goto unlock;
+        }
 
-	return 0;
+        ret = __afr_selfheal_data_read_write(
+            frame, this, fd, source, healed_sinks, offset, size, replies, type);
+    }
+unlock:
+    afr_selfheal_uninodelk(frame, this, fd->inode, this->name, offset, size,
+                           data_lock);
+    return ret;
 }
 
-
-int
-afr_sh_data_erase_pending_cbk (call_frame_t *frame, void *cookie,
-			       xlator_t *this, int32_t op_ret,
-			       int32_t op_errno, dict_t *xattr)
+static int
+afr_selfheal_data_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                        unsigned char *healed_sinks)
 {
-	int             call_count = 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
 
-	call_count = afr_frame_return (frame);
+    local = frame->local;
+    priv = this->private;
 
-	if (call_count == 0)
-		afr_sh_data_finish (frame, this);
-
-	return 0;
-}
+    if (!priv->ensure_durability)
+        return 0;
 
+    AFR_ONLIST(healed_sinks, frame, afr_sh_generic_fop_cbk, fsync, fd, 0, NULL);
 
-int
-afr_sh_data_erase_pending (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-	int              call_count = 0;
-	int              i = 0;
-	dict_t          **erase_xattr = NULL;
-
-
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
-
-	afr_sh_pending_to_delta (priv, sh->xattr, sh->delta_matrix, sh->success,
-                                 priv->child_count, AFR_DATA_TRANSACTION);
-
-	erase_xattr = GF_CALLOC (sizeof (*erase_xattr), priv->child_count,
-                              gf_afr_mt_dict_t);
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (sh->xattr[i]) {
-			call_count++;
-
-			erase_xattr[i] = get_new_dict();
-			dict_ref (erase_xattr[i]);
-		}
-	}
-
-	afr_sh_delta_to_xattr (priv, sh->delta_matrix, erase_xattr,
-			       priv->child_count, AFR_DATA_TRANSACTION);
-
-	local->call_count = call_count;
-	for (i = 0; i < priv->child_count; i++) {
-		if (!erase_xattr[i])
-			continue;
-
-		gf_log (this->name, GF_LOG_TRACE,
-			"erasing pending flags from %s on %s",
-			local->loc.path, priv->children[i]->name);
-
-		STACK_WIND_COOKIE (frame, afr_sh_data_erase_pending_cbk,
-				   (void *) (long) i,
-				   priv->children[i],
-				   priv->children[i]->fops->fxattrop,
-				   sh->healing_fd,
-				   GF_XATTROP_ADD_ARRAY, erase_xattr[i]);
-		if (!--call_count)
-			break;
-	}
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (erase_xattr[i]) {
-			dict_unref (erase_xattr[i]);
-		}
-	}
-	GF_FREE (erase_xattr);
-
-	return 0;
+    for (i = 0; i < priv->child_count; i++)
+        if (healed_sinks[i] && local->replies[i].op_ret != 0)
+            /* fsync() failed. Do NOT consider this server
+               as successfully healed. Mark it so.
+            */
+            healed_sinks[i] = 0;
+    return 0;
 }
 
-
-int
-afr_sh_data_trim_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                      struct iatt *postbuf)
+static int
+afr_data_self_heal_type_get(afr_private_t *priv, unsigned char *healed_sinks,
+                            int source, struct afr_reply *replies)
 {
-	afr_private_t * priv = NULL;
-	afr_local_t * local  = NULL;
-	int              call_count = 0;
-	int              child_index = 0;
-
-	priv = this->private;
-	local = frame->local;
-
-	child_index = (long) cookie;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret == -1)
-			gf_log (this->name, GF_LOG_DEBUG,
-				"ftruncate of %s on subvolume %s failed (%s)",
-				local->loc.path,
-				priv->children[child_index]->name,
-				strerror (op_errno));
-		else
-			gf_log (this->name, GF_LOG_TRACE,
-				"ftruncate of %s on subvolume %s completed",
-				local->loc.path,
-				priv->children[child_index]->name);
-	}
-	UNLOCK (&frame->lock);
-
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0) {
-		afr_sh_data_erase_pending (frame, this);
-	}
-
-	return 0;
+    int type = AFR_SELFHEAL_DATA_FULL;
+    int i = 0;
+
+    if (priv->data_self_heal_algorithm == AFR_SELFHEAL_DATA_DYNAMIC) {
+        type = AFR_SELFHEAL_DATA_FULL;
+        for (i = 0; i < priv->child_count; i++) {
+            if (!healed_sinks[i] && i != source)
+                continue;
+            if (replies[i].poststat.ia_size) {
+                type = AFR_SELFHEAL_DATA_DIFF;
+                break;
+            }
+        }
+    } else {
+        type = priv->data_self_heal_algorithm;
+    }
+    return type;
 }
 
-
-int
-afr_sh_data_trim_sinks (call_frame_t *frame, xlator_t *this)
+static int
+afr_selfheal_data_do(call_frame_t *frame, xlator_t *this, fd_t *fd, int source,
+                     unsigned char *healed_sinks, struct afr_reply *replies)
 {
-	afr_private_t * priv = NULL;
-	afr_local_t * local  = NULL;
-	afr_self_heal_t *sh  = NULL;
-	int             *sources = NULL;
-	int              call_count = 0;
-	int              i = 0;
-
-
-	priv = this->private;
-	local = frame->local;
-	sh = &local->self_heal;
-
-	sources = sh->sources;
-	call_count = sh->active_sinks;
-
-	local->call_count = call_count;
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (sources[i] || !local->child_up[i])
-			continue;
-
-		STACK_WIND_COOKIE (frame, afr_sh_data_trim_cbk,
-				   (void *) (long) i,
-				   priv->children[i], 
-				   priv->children[i]->fops->ftruncate,
-				   sh->healing_fd, sh->file_size); 
-
-		if (!--call_count)
-			break;
-	}
-
-	return 0;
-}
+    afr_private_t *priv = NULL;
+    off_t off = 0;
+    size_t block = 0;
+    int type = AFR_SELFHEAL_DATA_FULL;
+    int ret = -1;
+    call_frame_t *iter_frame = NULL;
+    unsigned char arbiter_sink_status = 0;
+
+    gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_SELF_HEAL_INFO,
+           "performing data selfheal on %s", uuid_utoa(fd->inode->gfid));
+
+    priv = this->private;
+    if (priv->arbiter_count) {
+        arbiter_sink_status = healed_sinks[ARBITER_BRICK_INDEX];
+        healed_sinks[ARBITER_BRICK_INDEX] = 0;
+    }
+
+    block = 128 * 1024 * priv->data_self_heal_window_size;
+
+    type = afr_data_self_heal_type_get(priv, healed_sinks, source, replies);
+
+    iter_frame = afr_copy_frame(frame);
+    if (!iter_frame) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    for (off = 0; off < replies[source].poststat.ia_size; off += block) {
+        if (AFR_COUNT(healed_sinks, priv->child_count) == 0) {
+            ret = -ENOTCONN;
+            goto out;
+        }